2024-12-09 08:20:38,245 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 08:20:38,256 main DEBUG Took 0.009482 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 08:20:38,257 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 08:20:38,257 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 08:20:38,258 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 08:20:38,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,266 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 08:20:38,276 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,278 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,279 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,280 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,281 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,282 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,283 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,284 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,285 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,285 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,286 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:20:38,286 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,287 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 08:20:38,288 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:20:38,289 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 08:20:38,291 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 08:20:38,291 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 08:20:38,292 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 08:20:38,293 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 08:20:38,300 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 08:20:38,303 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 08:20:38,304 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 08:20:38,304 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 08:20:38,305 main DEBUG createAppenders(={Console}) 2024-12-09 08:20:38,305 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-09 08:20:38,305 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 08:20:38,306 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-09 08:20:38,306 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 08:20:38,306 main DEBUG OutputStream closed 2024-12-09 08:20:38,307 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 08:20:38,307 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 08:20:38,307 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-09 08:20:38,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 08:20:38,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 08:20:38,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 08:20:38,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 08:20:38,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 08:20:38,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 08:20:38,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 08:20:38,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 08:20:38,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 08:20:38,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 08:20:38,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 08:20:38,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 08:20:38,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 08:20:38,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 08:20:38,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 08:20:38,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 08:20:38,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 08:20:38,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 08:20:38,379 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 08:20:38,379 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-09 08:20:38,379 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 08:20:38,380 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-09T08:20:38,589 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c 2024-12-09 08:20:38,592 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 08:20:38,592 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T08:20:38,600 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T08:20:38,606 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-09T08:20:38,625 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T08:20:38,665 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T08:20:38,665 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T08:20:38,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T08:20:38,694 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff, deleteOnExit=true 2024-12-09T08:20:38,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T08:20:38,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/test.cache.data in system properties and HBase conf 2024-12-09T08:20:38,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T08:20:38,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir in system properties and HBase conf 2024-12-09T08:20:38,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T08:20:38,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T08:20:38,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T08:20:38,772 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T08:20:38,866 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T08:20:38,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T08:20:38,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T08:20:38,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T08:20:38,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T08:20:38,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T08:20:38,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T08:20:38,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T08:20:38,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T08:20:38,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T08:20:38,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/nfs.dump.dir in system properties and HBase conf 2024-12-09T08:20:38,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/java.io.tmpdir in system properties and HBase conf 2024-12-09T08:20:38,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T08:20:38,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T08:20:38,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T08:20:39,914 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T08:20:39,979 INFO [Time-limited test {}] log.Log(170): Logging initialized @2353ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T08:20:40,045 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:20:40,108 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:20:40,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:20:40,134 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:20:40,135 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:20:40,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:20:40,154 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:20:40,155 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:20:40,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/java.io.tmpdir/jetty-localhost-36041-hadoop-hdfs-3_4_1-tests_jar-_-any-1611480492064370701/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T08:20:40,349 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:36041} 2024-12-09T08:20:40,349 INFO [Time-limited test {}] server.Server(415): Started @2722ms 2024-12-09T08:20:40,960 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:20:40,968 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:20:40,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:20:40,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:20:40,970 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:20:40,970 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:20:40,971 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:20:41,067 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/java.io.tmpdir/jetty-localhost-35501-hadoop-hdfs-3_4_1-tests_jar-_-any-6709143594556534207/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:20:41,067 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:35501} 2024-12-09T08:20:41,067 INFO [Time-limited test {}] server.Server(415): Started @3441ms 2024-12-09T08:20:41,112 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:20:41,224 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:20:41,229 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:20:41,230 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:20:41,230 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:20:41,230 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:20:41,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:20:41,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:20:41,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/java.io.tmpdir/jetty-localhost-37267-hadoop-hdfs-3_4_1-tests_jar-_-any-1424562202187386743/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:20:41,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:37267} 2024-12-09T08:20:41,330 INFO [Time-limited test {}] server.Server(415): Started @3704ms 2024-12-09T08:20:41,332 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:20:41,374 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:20:41,379 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:20:41,380 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:20:41,381 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:20:41,381 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T08:20:41,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:20:41,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:20:41,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/java.io.tmpdir/jetty-localhost-40721-hadoop-hdfs-3_4_1-tests_jar-_-any-12707239087861787445/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:20:41,497 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:40721} 2024-12-09T08:20:41,498 INFO [Time-limited test {}] server.Server(415): Started @3871ms 2024-12-09T08:20:41,501 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:20:42,738 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data4/current/BP-361248767-172.17.0.2-1733732439363/current, will proceed with Du for space computation calculation, 2024-12-09T08:20:42,738 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data2/current/BP-361248767-172.17.0.2-1733732439363/current, will proceed with Du for space computation calculation, 2024-12-09T08:20:42,738 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data3/current/BP-361248767-172.17.0.2-1733732439363/current, will proceed with Du for space computation calculation, 2024-12-09T08:20:42,738 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data1/current/BP-361248767-172.17.0.2-1733732439363/current, will proceed with Du for space computation calculation, 2024-12-09T08:20:42,777 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:20:42,781 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:20:42,821 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd35badae9ad71572 with lease ID 0xf1eb2462f742a2b4: Processing first storage report for DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4 from datanode DatanodeRegistration(127.0.0.1:40851, datanodeUuid=efcf3e6d-2d12-4da2-b536-d812b4be17aa, infoPort=35705, infoSecurePort=0, ipcPort=43249, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363) 2024-12-09T08:20:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd35badae9ad71572 with lease ID 0xf1eb2462f742a2b4: from storage DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4 node DatanodeRegistration(127.0.0.1:40851, datanodeUuid=efcf3e6d-2d12-4da2-b536-d812b4be17aa, infoPort=35705, infoSecurePort=0, ipcPort=43249, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T08:20:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xff69974a78e8b827 with lease ID 0xf1eb2462f742a2b5: Processing first storage report for DS-6ff8f618-13fb-4fe5-8163-398e337dc475 from datanode DatanodeRegistration(127.0.0.1:38773, datanodeUuid=581cf5ca-7eed-440c-9140-bcdd1fc65c2c, infoPort=45923, infoSecurePort=0, ipcPort=43985, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363) 2024-12-09T08:20:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xff69974a78e8b827 with lease ID 0xf1eb2462f742a2b5: from storage DS-6ff8f618-13fb-4fe5-8163-398e337dc475 node DatanodeRegistration(127.0.0.1:38773, datanodeUuid=581cf5ca-7eed-440c-9140-bcdd1fc65c2c, infoPort=45923, infoSecurePort=0, ipcPort=43985, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T08:20:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd35badae9ad71572 with lease ID 0xf1eb2462f742a2b4: Processing first storage report for DS-c1be1433-4011-460f-bd8c-fe568152b034 from datanode DatanodeRegistration(127.0.0.1:40851, datanodeUuid=efcf3e6d-2d12-4da2-b536-d812b4be17aa, infoPort=35705, infoSecurePort=0, ipcPort=43249, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363) 2024-12-09T08:20:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd35badae9ad71572 with lease ID 0xf1eb2462f742a2b4: from storage DS-c1be1433-4011-460f-bd8c-fe568152b034 node DatanodeRegistration(127.0.0.1:40851, datanodeUuid=efcf3e6d-2d12-4da2-b536-d812b4be17aa, infoPort=35705, infoSecurePort=0, ipcPort=43249, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:20:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xff69974a78e8b827 with lease ID 0xf1eb2462f742a2b5: Processing first storage report for DS-b42973a3-24f2-409f-aca5-856d97364a99 from datanode DatanodeRegistration(127.0.0.1:38773, datanodeUuid=581cf5ca-7eed-440c-9140-bcdd1fc65c2c, infoPort=45923, infoSecurePort=0, ipcPort=43985, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363) 2024-12-09T08:20:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xff69974a78e8b827 with lease ID 0xf1eb2462f742a2b5: from storage DS-b42973a3-24f2-409f-aca5-856d97364a99 node DatanodeRegistration(127.0.0.1:38773, datanodeUuid=581cf5ca-7eed-440c-9140-bcdd1fc65c2c, infoPort=45923, infoSecurePort=0, ipcPort=43985, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:20:42,832 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data6/current/BP-361248767-172.17.0.2-1733732439363/current, will proceed with Du for space computation calculation, 2024-12-09T08:20:42,832 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data5/current/BP-361248767-172.17.0.2-1733732439363/current, will proceed with Du for space computation calculation, 2024-12-09T08:20:42,855 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:20:42,859 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf761b8661e01878f with lease ID 0xf1eb2462f742a2b6: Processing first storage report for DS-cdb20101-1a35-4f28-b07d-ecf312072bf9 from datanode DatanodeRegistration(127.0.0.1:41795, datanodeUuid=0a71e0a5-6836-4c09-b525-c241270ea2a3, infoPort=33395, infoSecurePort=0, ipcPort=39947, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363) 2024-12-09T08:20:42,859 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf761b8661e01878f with lease ID 0xf1eb2462f742a2b6: from storage DS-cdb20101-1a35-4f28-b07d-ecf312072bf9 node DatanodeRegistration(127.0.0.1:41795, datanodeUuid=0a71e0a5-6836-4c09-b525-c241270ea2a3, infoPort=33395, infoSecurePort=0, ipcPort=39947, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:20:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf761b8661e01878f with lease ID 0xf1eb2462f742a2b6: Processing first storage report for DS-73e29d6d-232d-45f9-9b0a-ac614ea4775f from datanode DatanodeRegistration(127.0.0.1:41795, datanodeUuid=0a71e0a5-6836-4c09-b525-c241270ea2a3, infoPort=33395, infoSecurePort=0, ipcPort=39947, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363) 2024-12-09T08:20:42,860 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf761b8661e01878f with lease ID 0xf1eb2462f742a2b6: from storage DS-73e29d6d-232d-45f9-9b0a-ac614ea4775f node DatanodeRegistration(127.0.0.1:41795, datanodeUuid=0a71e0a5-6836-4c09-b525-c241270ea2a3, infoPort=33395, infoSecurePort=0, ipcPort=39947, storageInfo=lv=-57;cid=testClusterID;nsid=761542811;c=1733732439363), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:20:42,906 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c 2024-12-09T08:20:42,965 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/zookeeper_0, clientPort=61985, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T08:20:42,983 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61985 2024-12-09T08:20:43,000 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:43,003 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:43,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:20:43,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:20:43,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:20:43,581 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609 with version=8 2024-12-09T08:20:43,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/hbase-staging 2024-12-09T08:20:43,878 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:20:43,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:43,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:43,892 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:20:43,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:43,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:20:44,031 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T08:20:44,093 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T08:20:44,103 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T08:20:44,107 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:20:44,132 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 59587 (auto-detected) 2024-12-09T08:20:44,133 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T08:20:44,151 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40865 2024-12-09T08:20:44,171 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40865 connecting to ZooKeeper ensemble=127.0.0.1:61985 2024-12-09T08:20:44,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:408650x0, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:20:44,309 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40865-0x10009a9aa0d0000 connected 2024-12-09T08:20:44,400 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,403 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:20:44,416 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609, hbase.cluster.distributed=false 2024-12-09T08:20:44,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:20:44,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40865 2024-12-09T08:20:44,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40865 2024-12-09T08:20:44,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40865 2024-12-09T08:20:44,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40865 2024-12-09T08:20:44,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40865 2024-12-09T08:20:44,573 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:20:44,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,575 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:20:44,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:20:44,578 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:20:44,580 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:20:44,581 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41239 2024-12-09T08:20:44,583 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41239 connecting to ZooKeeper ensemble=127.0.0.1:61985 2024-12-09T08:20:44,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,587 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412390x0, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:20:44,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412390x0, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:20:44,599 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41239-0x10009a9aa0d0001 connected 2024-12-09T08:20:44,604 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:20:44,614 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:20:44,617 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:20:44,624 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:20:44,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41239 2024-12-09T08:20:44,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41239 2024-12-09T08:20:44,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41239 2024-12-09T08:20:44,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41239 2024-12-09T08:20:44,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41239 2024-12-09T08:20:44,648 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:20:44,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,649 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:20:44,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:20:44,649 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:20:44,650 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:20:44,651 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33703 2024-12-09T08:20:44,653 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33703 connecting to ZooKeeper ensemble=127.0.0.1:61985 2024-12-09T08:20:44,654 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,672 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337030x0, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:20:44,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:337030x0, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:20:44,673 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33703-0x10009a9aa0d0002 connected 2024-12-09T08:20:44,673 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:20:44,674 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:20:44,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:20:44,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:20:44,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33703 2024-12-09T08:20:44,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33703 2024-12-09T08:20:44,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33703 2024-12-09T08:20:44,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33703 2024-12-09T08:20:44,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33703 2024-12-09T08:20:44,708 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:20:44,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,709 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:20:44,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:20:44,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:20:44,709 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:20:44,709 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:20:44,710 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41695 2024-12-09T08:20:44,713 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41695 connecting to ZooKeeper ensemble=127.0.0.1:61985 2024-12-09T08:20:44,714 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,733 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416950x0, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:20:44,734 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41695-0x10009a9aa0d0003 connected 2024-12-09T08:20:44,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:20:44,735 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:20:44,736 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:20:44,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:20:44,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:20:44,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41695 2024-12-09T08:20:44,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41695 2024-12-09T08:20:44,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41695 2024-12-09T08:20:44,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41695 2024-12-09T08:20:44,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41695 2024-12-09T08:20:44,754 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5bda946059d1:40865 2024-12-09T08:20:44,755 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5bda946059d1,40865,1733732443726 2024-12-09T08:20:44,767 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,767 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,769 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5bda946059d1,40865,1733732443726 2024-12-09T08:20:44,798 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:20:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:20:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,798 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:20:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,798 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,799 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,799 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T08:20:44,801 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5bda946059d1,40865,1733732443726 from backup master directory 2024-12-09T08:20:44,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5bda946059d1,40865,1733732443726 2024-12-09T08:20:44,808 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,808 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:20:44,810 WARN [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:20:44,810 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5bda946059d1,40865,1733732443726 2024-12-09T08:20:44,812 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T08:20:44,814 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T08:20:44,864 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/hbase.id] with ID: b0508df1-95ca-4311-b15c-3eff9ce222f2 2024-12-09T08:20:44,865 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/.tmp/hbase.id 2024-12-09T08:20:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:20:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:20:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:20:44,878 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/.tmp/hbase.id]:[hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/hbase.id] 2024-12-09T08:20:44,924 INFO [master/5bda946059d1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:20:44,929 INFO [master/5bda946059d1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T08:20:44,945 INFO [master/5bda946059d1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-09T08:20:44,956 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,956 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:44,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:20:44,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:20:44,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:20:44,993 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:20:44,995 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T08:20:45,002 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:45,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:20:45,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:20:45,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:20:45,057 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store 2024-12-09T08:20:45,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:20:45,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:20:45,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:20:45,087 INFO [master/5bda946059d1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T08:20:45,090 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:45,091 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T08:20:45,091 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:20:45,091 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:20:45,093 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T08:20:45,093 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:20:45,093 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:20:45,094 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733732445091Disabling compacts and flushes for region at 1733732445091Disabling writes for close at 1733732445093 (+2 ms)Writing region close event to WAL at 1733732445093Closed at 1733732445093 2024-12-09T08:20:45,096 WARN [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/.initializing 2024-12-09T08:20:45,097 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/WALs/5bda946059d1,40865,1733732443726 2024-12-09T08:20:45,106 INFO [master/5bda946059d1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:45,120 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C40865%2C1733732443726, suffix=, logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/WALs/5bda946059d1,40865,1733732443726, archiveDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/oldWALs, maxLogs=10 2024-12-09T08:20:45,149 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/WALs/5bda946059d1,40865,1733732443726/5bda946059d1%2C40865%2C1733732443726.1733732445124, exclude list is [], retry=0 2024-12-09T08:20:45,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:45,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:45,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:45,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:45,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T08:20:45,214 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/WALs/5bda946059d1,40865,1733732443726/5bda946059d1%2C40865%2C1733732443726.1733732445124 2024-12-09T08:20:45,215 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:45,216 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:45,216 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:45,219 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,220 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T08:20:45,277 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:45,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:45,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T08:20:45,284 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:45,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:45,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T08:20:45,288 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:45,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:45,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T08:20:45,292 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:45,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:45,294 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,298 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,299 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,304 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,305 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,309 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:20:45,312 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:20:45,317 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:45,319 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59464951, jitterRate=-0.11390317976474762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:20:45,324 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733732445231Initializing all the Stores at 1733732445233 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732445233Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732445234 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732445234Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732445234Cleaning up temporary data from old regions at 1733732445305 (+71 ms)Region opened successfully at 1733732445324 (+19 ms) 2024-12-09T08:20:45,325 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T08:20:45,357 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21669533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:20:45,387 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T08:20:45,397 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T08:20:45,398 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T08:20:45,400 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T08:20:45,401 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T08:20:45,406 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T08:20:45,406 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T08:20:45,428 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T08:20:45,436 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T08:20:45,514 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T08:20:45,517 INFO [master/5bda946059d1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T08:20:45,519 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T08:20:45,591 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T08:20:45,594 INFO [master/5bda946059d1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T08:20:45,597 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T08:20:45,628 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T08:20:45,630 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T08:20:45,640 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T08:20:45,656 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T08:20:45,661 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T08:20:45,671 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:20:45,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:20:45,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:20:45,672 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,671 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:20:45,672 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,675 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5bda946059d1,40865,1733732443726, sessionid=0x10009a9aa0d0000, setting cluster-up flag (Was=false) 2024-12-09T08:20:45,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,703 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,703 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,735 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T08:20:45,737 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5bda946059d1,40865,1733732443726 2024-12-09T08:20:45,756 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,756 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:45,787 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T08:20:45,789 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5bda946059d1,40865,1733732443726 2024-12-09T08:20:45,796 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T08:20:45,845 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(746): ClusterId : b0508df1-95ca-4311-b15c-3eff9ce222f2 2024-12-09T08:20:45,845 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(746): ClusterId : b0508df1-95ca-4311-b15c-3eff9ce222f2 2024-12-09T08:20:45,846 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(746): ClusterId : b0508df1-95ca-4311-b15c-3eff9ce222f2 2024-12-09T08:20:45,847 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:20:45,847 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:20:45,847 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:20:45,870 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T08:20:45,873 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:20:45,873 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:20:45,873 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:20:45,873 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:20:45,873 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:20:45,873 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:20:45,879 INFO [master/5bda946059d1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T08:20:45,887 INFO [master/5bda946059d1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T08:20:45,893 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:20:45,893 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:20:45,893 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:20:45,894 DEBUG [RS:1;5bda946059d1:33703 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb75b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:20:45,894 DEBUG [RS:2;5bda946059d1:41695 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f8b367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:20:45,894 DEBUG [RS:0;5bda946059d1:41239 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44658ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:20:45,893 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5bda946059d1,40865,1733732443726 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T08:20:45,903 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:20:45,903 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:20:45,903 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:20:45,903 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:20:45,904 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5bda946059d1:0, corePoolSize=10, maxPoolSize=10 2024-12-09T08:20:45,904 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:45,904 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:20:45,904 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:45,908 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;5bda946059d1:33703 2024-12-09T08:20:45,912 INFO [RS:1;5bda946059d1:33703 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:20:45,912 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5bda946059d1:41239 2024-12-09T08:20:45,912 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;5bda946059d1:41695 2024-12-09T08:20:45,912 INFO [RS:1;5bda946059d1:33703 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:20:45,912 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733732475912 2024-12-09T08:20:45,913 INFO [RS:0;5bda946059d1:41239 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:20:45,913 INFO [RS:0;5bda946059d1:41239 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:20:45,913 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:20:45,913 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:20:45,913 INFO [RS:2;5bda946059d1:41695 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:20:45,913 INFO [RS:2;5bda946059d1:41695 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:20:45,913 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:20:45,914 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T08:20:45,916 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T08:20:45,916 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,40865,1733732443726 with port=33703, startcode=1733732444647 2024-12-09T08:20:45,916 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,40865,1733732443726 with port=41239, startcode=1733732444531 2024-12-09T08:20:45,916 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:20:45,916 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,40865,1733732443726 with port=41695, startcode=1733732444707 2024-12-09T08:20:45,917 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T08:20:45,920 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T08:20:45,920 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T08:20:45,921 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T08:20:45,921 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T08:20:45,924 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:45,925 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T08:20:45,931 DEBUG [RS:0;5bda946059d1:41239 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:20:45,931 DEBUG [RS:1;5bda946059d1:33703 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:20:45,929 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:45,932 DEBUG [RS:2;5bda946059d1:41695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:20:45,932 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T08:20:45,933 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T08:20:45,934 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T08:20:45,937 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T08:20:45,937 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T08:20:45,939 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732445938,5,FailOnTimeoutGroup] 2024-12-09T08:20:45,941 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732445939,5,FailOnTimeoutGroup] 2024-12-09T08:20:45,941 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:45,941 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T08:20:45,942 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:45,942 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:45,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:20:45,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:20:45,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:20:45,983 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T08:20:45,983 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609 2024-12-09T08:20:45,999 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58047, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:20:45,999 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42263, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:20:45,999 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51207, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:20:46,005 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,41239,1733732444531 2024-12-09T08:20:46,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741832_1008 (size=32) 2024-12-09T08:20:46,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741832_1008 (size=32) 2024-12-09T08:20:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741832_1008 (size=32) 2024-12-09T08:20:46,009 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40865 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,41239,1733732444531 2024-12-09T08:20:46,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:46,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T08:20:46,018 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T08:20:46,018 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T08:20:46,023 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T08:20:46,023 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,024 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,41695,1733732444707 2024-12-09T08:20:46,024 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40865 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,41695,1733732444707 2024-12-09T08:20:46,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T08:20:46,028 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609 2024-12-09T08:20:46,028 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40523 2024-12-09T08:20:46,028 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:20:46,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T08:20:46,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,029 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,029 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40865 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T08:20:46,031 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609 2024-12-09T08:20:46,031 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40523 2024-12-09T08:20:46,032 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:20:46,034 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609 2024-12-09T08:20:46,034 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T08:20:46,034 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40523 2024-12-09T08:20:46,034 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,034 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:20:46,035 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T08:20:46,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740 2024-12-09T08:20:46,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740 2024-12-09T08:20:46,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T08:20:46,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T08:20:46,044 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:20:46,047 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T08:20:46,052 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:46,053 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72920562, jitterRate=0.08660104870796204}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:20:46,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733732446012Initializing all the Stores at 1733732446014 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732446014Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732446014Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732446014Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732446014Cleaning up temporary data from old regions at 1733732446043 (+29 ms)Region opened successfully at 1733732446056 (+13 ms) 2024-12-09T08:20:46,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T08:20:46,056 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T08:20:46,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T08:20:46,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T08:20:46,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T08:20:46,058 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T08:20:46,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733732446056Disabling compacts and flushes for region at 1733732446056Disabling writes for close at 1733732446057 (+1 ms)Writing region close event to WAL at 1733732446058 (+1 ms)Closed at 1733732446058 2024-12-09T08:20:46,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:20:46,062 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:20:46,062 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T08:20:46,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T08:20:46,079 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T08:20:46,082 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T08:20:46,103 DEBUG [RS:0;5bda946059d1:41239 {}] zookeeper.ZKUtil(111): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,41239,1733732444531 2024-12-09T08:20:46,103 WARN [RS:0;5bda946059d1:41239 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:20:46,103 INFO [RS:0;5bda946059d1:41239 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:46,103 DEBUG [RS:1;5bda946059d1:33703 {}] zookeeper.ZKUtil(111): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,103 DEBUG [RS:2;5bda946059d1:41695 {}] zookeeper.ZKUtil(111): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,41695,1733732444707 2024-12-09T08:20:46,103 WARN [RS:1;5bda946059d1:33703 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:20:46,103 WARN [RS:2;5bda946059d1:41695 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:20:46,103 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41239,1733732444531 2024-12-09T08:20:46,103 INFO [RS:2;5bda946059d1:41695 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:46,103 INFO [RS:1;5bda946059d1:33703 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:46,104 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,104 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707 2024-12-09T08:20:46,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,41239,1733732444531] 2024-12-09T08:20:46,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,33703,1733732444647] 2024-12-09T08:20:46,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,41695,1733732444707] 2024-12-09T08:20:46,132 INFO [RS:1;5bda946059d1:33703 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:20:46,132 INFO [RS:2;5bda946059d1:41695 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:20:46,132 INFO [RS:0;5bda946059d1:41239 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:20:46,148 INFO [RS:2;5bda946059d1:41695 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:20:46,148 INFO [RS:0;5bda946059d1:41239 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:20:46,148 INFO [RS:1;5bda946059d1:33703 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:20:46,153 INFO [RS:0;5bda946059d1:41239 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:20:46,153 INFO [RS:1;5bda946059d1:33703 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:20:46,153 INFO [RS:2;5bda946059d1:41695 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:20:46,154 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,154 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,154 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,155 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:20:46,155 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:20:46,159 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:20:46,161 INFO [RS:2;5bda946059d1:41695 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:20:46,161 INFO [RS:0;5bda946059d1:41239 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:20:46,161 INFO [RS:1;5bda946059d1:33703 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:20:46,162 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,162 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,163 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,163 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,163 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:20:46,164 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:20:46,164 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,164 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:20:46,165 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:20:46,165 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:20:46,165 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:20:46,165 DEBUG [RS:1;5bda946059d1:33703 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:20:46,165 DEBUG [RS:0;5bda946059d1:41239 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:20:46,165 DEBUG [RS:2;5bda946059d1:41695 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:20:46,175 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,175 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,33703,1733732444647-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41695,1733732444707-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:20:46,176 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,178 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,178 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41239,1733732444531-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:20:46,202 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:20:46,202 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:20:46,202 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:20:46,205 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41239,1733732444531-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,205 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41695,1733732444707-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,205 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,33703,1733732444647-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,205 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,205 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,205 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,205 INFO [RS:0;5bda946059d1:41239 {}] regionserver.Replication(171): 5bda946059d1,41239,1733732444531 started 2024-12-09T08:20:46,205 INFO [RS:1;5bda946059d1:33703 {}] regionserver.Replication(171): 5bda946059d1,33703,1733732444647 started 2024-12-09T08:20:46,205 INFO [RS:2;5bda946059d1:41695 {}] regionserver.Replication(171): 5bda946059d1,41695,1733732444707 started 2024-12-09T08:20:46,229 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,229 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,229 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:46,230 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,41695,1733732444707, RpcServer on 5bda946059d1/172.17.0.2:41695, sessionid=0x10009a9aa0d0003 2024-12-09T08:20:46,230 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,41239,1733732444531, RpcServer on 5bda946059d1/172.17.0.2:41239, sessionid=0x10009a9aa0d0001 2024-12-09T08:20:46,230 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,33703,1733732444647, RpcServer on 5bda946059d1/172.17.0.2:33703, sessionid=0x10009a9aa0d0002 2024-12-09T08:20:46,231 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:20:46,231 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:20:46,231 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:20:46,231 DEBUG [RS:1;5bda946059d1:33703 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,231 DEBUG [RS:2;5bda946059d1:41695 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,41695,1733732444707 2024-12-09T08:20:46,231 DEBUG [RS:0;5bda946059d1:41239 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,41239,1733732444531 2024-12-09T08:20:46,231 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,41239,1733732444531' 2024-12-09T08:20:46,231 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,41695,1733732444707' 2024-12-09T08:20:46,231 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,33703,1733732444647' 2024-12-09T08:20:46,231 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:20:46,231 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:20:46,231 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:20:46,233 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:20:46,233 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:20:46,233 WARN [5bda946059d1:40865 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T08:20:46,233 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:20:46,233 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:20:46,233 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:20:46,234 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:20:46,234 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:20:46,234 DEBUG [RS:2;5bda946059d1:41695 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,41695,1733732444707 2024-12-09T08:20:46,234 DEBUG [RS:1;5bda946059d1:33703 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,234 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,41695,1733732444707' 2024-12-09T08:20:46,234 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,33703,1733732444647' 2024-12-09T08:20:46,234 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:20:46,234 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:20:46,234 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:20:46,234 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:20:46,234 DEBUG [RS:0;5bda946059d1:41239 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,41239,1733732444531 2024-12-09T08:20:46,234 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,41239,1733732444531' 2024-12-09T08:20:46,234 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:20:46,235 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:20:46,235 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:20:46,235 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:20:46,235 DEBUG [RS:2;5bda946059d1:41695 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:20:46,235 INFO [RS:2;5bda946059d1:41695 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:20:46,235 INFO [RS:2;5bda946059d1:41695 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:20:46,236 DEBUG [RS:1;5bda946059d1:33703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:20:46,236 INFO [RS:1;5bda946059d1:33703 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:20:46,236 INFO [RS:1;5bda946059d1:33703 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:20:46,236 DEBUG [RS:0;5bda946059d1:41239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:20:46,236 INFO [RS:0;5bda946059d1:41239 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:20:46,236 INFO [RS:0;5bda946059d1:41239 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:20:46,340 INFO [RS:2;5bda946059d1:41695 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:46,340 INFO [RS:0;5bda946059d1:41239 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:46,340 INFO [RS:1;5bda946059d1:33703 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:46,342 INFO [RS:0;5bda946059d1:41239 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C41239%2C1733732444531, suffix=, logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41239,1733732444531, archiveDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs, maxLogs=32 2024-12-09T08:20:46,342 INFO [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C33703%2C1733732444647, suffix=, logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647, archiveDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs, maxLogs=32 2024-12-09T08:20:46,342 INFO [RS:2;5bda946059d1:41695 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C41695%2C1733732444707, suffix=, logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707, archiveDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs, maxLogs=32 2024-12-09T08:20:46,357 DEBUG [RS:0;5bda946059d1:41239 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41239,1733732444531/5bda946059d1%2C41239%2C1733732444531.1733732446345, exclude list is [], retry=0 2024-12-09T08:20:46,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:46,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:46,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:46,364 DEBUG [RS:2;5bda946059d1:41695 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707/5bda946059d1%2C41695%2C1733732444707.1733732446346, exclude list is [], retry=0 2024-12-09T08:20:46,364 DEBUG [RS:1;5bda946059d1:33703 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647/5bda946059d1%2C33703%2C1733732444647.1733732446346, exclude list is [], retry=0 2024-12-09T08:20:46,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:46,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:46,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:46,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:46,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:46,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:46,388 INFO [RS:0;5bda946059d1:41239 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41239,1733732444531/5bda946059d1%2C41239%2C1733732444531.1733732446345 2024-12-09T08:20:46,395 DEBUG [RS:0;5bda946059d1:41239 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:46,396 INFO [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647/5bda946059d1%2C33703%2C1733732444647.1733732446346 2024-12-09T08:20:46,397 INFO [RS:2;5bda946059d1:41695 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707/5bda946059d1%2C41695%2C1733732444707.1733732446346 2024-12-09T08:20:46,397 DEBUG [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:20:46,397 DEBUG [RS:2;5bda946059d1:41695 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:46,486 DEBUG [5bda946059d1:40865 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T08:20:46,492 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(204): Hosts are {5bda946059d1=0} racks are {/default-rack=0} 2024-12-09T08:20:46,498 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T08:20:46,498 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T08:20:46,498 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T08:20:46,499 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T08:20:46,499 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T08:20:46,499 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T08:20:46,499 INFO [5bda946059d1:40865 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T08:20:46,499 INFO [5bda946059d1:40865 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T08:20:46,499 INFO [5bda946059d1:40865 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T08:20:46,499 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T08:20:46,505 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,509 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5bda946059d1,33703,1733732444647, state=OPENING 2024-12-09T08:20:46,556 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T08:20:46,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:46,566 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:46,566 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:46,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:20:46,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,570 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T08:20:46,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5bda946059d1,33703,1733732444647}] 2024-12-09T08:20:46,744 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:20:46,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34857, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:20:46,758 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T08:20:46,759 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:46,759 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T08:20:46,762 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C33703%2C1733732444647.meta, suffix=.meta, logDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647, archiveDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs, maxLogs=32 2024-12-09T08:20:46,777 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647/5bda946059d1%2C33703%2C1733732444647.meta.1733732446763.meta, exclude list is [], retry=0 2024-12-09T08:20:46,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:46,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:46,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:46,784 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,33703,1733732444647/5bda946059d1%2C33703%2C1733732444647.meta.1733732446763.meta 2024-12-09T08:20:46,784 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:46,784 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:46,786 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T08:20:46,788 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T08:20:46,793 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T08:20:46,796 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T08:20:46,797 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:46,797 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T08:20:46,797 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T08:20:46,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T08:20:46,802 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T08:20:46,802 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T08:20:46,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T08:20:46,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T08:20:46,808 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T08:20:46,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T08:20:46,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T08:20:46,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:46,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:20:46,811 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T08:20:46,813 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740 2024-12-09T08:20:46,816 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740 2024-12-09T08:20:46,820 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T08:20:46,821 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T08:20:46,822 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:20:46,825 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T08:20:46,827 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67536046, jitterRate=0.0063655078411102295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:20:46,827 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T08:20:46,829 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733732446798Writing region info on filesystem at 1733732446798Initializing all the Stores at 1733732446800 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732446800Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732446800Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732446800Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732446800Cleaning up temporary data from old regions at 1733732446821 (+21 ms)Running coprocessor post-open hooks at 1733732446828 (+7 ms)Region opened successfully at 1733732446829 (+1 ms) 2024-12-09T08:20:46,837 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733732446737 2024-12-09T08:20:46,849 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T08:20:46,849 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T08:20:46,852 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,854 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5bda946059d1,33703,1733732444647, state=OPEN 2024-12-09T08:20:46,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:20:46,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:20:46,903 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:20:46,903 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:20:46,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:20:46,904 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5bda946059d1,33703,1733732444647 2024-12-09T08:20:46,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T08:20:46,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5bda946059d1,33703,1733732444647 in 332 msec 2024-12-09T08:20:46,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T08:20:46,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 841 msec 2024-12-09T08:20:46,917 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:20:46,917 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T08:20:46,937 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:20:46,938 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,33703,1733732444647, seqNum=-1] 2024-12-09T08:20:46,961 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:20:46,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44159, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:20:46,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1520 sec 2024-12-09T08:20:46,982 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733732446982, completionTime=-1 2024-12-09T08:20:46,985 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T08:20:46,985 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T08:20:47,009 INFO [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T08:20:47,009 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733732507009 2024-12-09T08:20:47,009 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733732567009 2024-12-09T08:20:47,009 INFO [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-09T08:20:47,011 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T08:20:47,036 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,40865,1733732443726-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,037 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,40865,1733732443726-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,037 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,40865,1733732443726-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,038 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5bda946059d1:40865, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,038 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,045 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,050 DEBUG [master/5bda946059d1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T08:20:47,078 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.267sec 2024-12-09T08:20:47,079 INFO [master/5bda946059d1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T08:20:47,081 INFO [master/5bda946059d1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T08:20:47,082 INFO [master/5bda946059d1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T08:20:47,082 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T08:20:47,083 INFO [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T08:20:47,083 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,40865,1733732443726-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:20:47,084 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,40865,1733732443726-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T08:20:47,091 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T08:20:47,092 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T08:20:47,093 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,40865,1733732443726-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:20:47,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef44f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:20:47,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,40865,-1 for getting cluster id 2024-12-09T08:20:47,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:20:47,167 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b0508df1-95ca-4311-b15c-3eff9ce222f2' 2024-12-09T08:20:47,170 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:20:47,170 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b0508df1-95ca-4311-b15c-3eff9ce222f2" 2024-12-09T08:20:47,172 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38014acc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:20:47,172 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,40865,-1] 2024-12-09T08:20:47,175 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:20:47,177 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:20:47,178 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:20:47,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4a9819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:20:47,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:20:47,190 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,33703,1733732444647, seqNum=-1] 2024-12-09T08:20:47,190 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:20:47,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:20:47,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5bda946059d1,40865,1733732443726 2024-12-09T08:20:47,215 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40523/hbase 2024-12-09T08:20:47,228 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=4492 2024-12-09T08:20:47,250 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:47,254 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:47,255 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:47,259 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05763334, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-05763334, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:47,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05763334/hregion-05763334.1733732447260, exclude list is [], retry=0 2024-12-09T08:20:47,278 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:47,278 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:47,279 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:47,284 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05763334/hregion-05763334.1733732447260 2024-12-09T08:20:47,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:20:47,285 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0d13634f38a8e7778980c4dd7e0b9c48, NAME => 'testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:20:47,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741838_1014 (size=64) 2024-12-09T08:20:47,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741838_1014 (size=64) 2024-12-09T08:20:47,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741838_1014 (size=64) 2024-12-09T08:20:47,298 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:47,299 INFO [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,302 INFO [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d13634f38a8e7778980c4dd7e0b9c48 columnFamilyName a 2024-12-09T08:20:47,302 DEBUG [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:47,302 INFO [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] regionserver.HStore(327): Store=0d13634f38a8e7778980c4dd7e0b9c48/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:47,303 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,304 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,305 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,308 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,311 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:47,312 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d13634f38a8e7778980c4dd7e0b9c48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61226467, jitterRate=-0.08765454590320587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:20:47,312 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d13634f38a8e7778980c4dd7e0b9c48: Writing region info on filesystem at 1733732447298Initializing all the Stores at 1733732447299 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732447299Cleaning up temporary data from old regions at 1733732447305 (+6 ms)Region opened successfully at 1733732447312 (+7 ms) 2024-12-09T08:20:47,313 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d13634f38a8e7778980c4dd7e0b9c48, disabling compactions & flushes 2024-12-09T08:20:47,313 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48. 2024-12-09T08:20:47,313 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48. 2024-12-09T08:20:47,313 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48. after waiting 0 ms 2024-12-09T08:20:47,313 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48. 2024-12-09T08:20:47,313 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48. 2024-12-09T08:20:47,313 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d13634f38a8e7778980c4dd7e0b9c48: Waiting for close lock at 1733732447313Disabling compacts and flushes for region at 1733732447313Disabling writes for close at 1733732447313Writing region close event to WAL at 1733732447313Closed at 1733732447313 2024-12-09T08:20:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741837_1013 (size=93) 2024-12-09T08:20:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741837_1013 (size=93) 2024-12-09T08:20:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741837_1013 (size=93) 2024-12-09T08:20:47,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:47,325 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05763334:(num 1733732447260) 2024-12-09T08:20:47,327 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:20:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741839_1015 (size=276) 2024-12-09T08:20:47,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741839_1015 (size=276) 2024-12-09T08:20:47,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741839_1015 (size=276) 2024-12-09T08:20:47,345 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:20:47,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741840_1016 (size=230) 2024-12-09T08:20:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741840_1016 (size=230) 2024-12-09T08:20:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741840_1016 (size=230) 2024-12-09T08:20:47,382 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1, size=276 (276bytes) 2024-12-09T08:20:47,383 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T08:20:47,383 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T08:20:47,383 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1 2024-12-09T08:20:47,388 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1 after 3ms 2024-12-09T08:20:47,397 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,398 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1 took 17ms 2024-12-09T08:20:47,403 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1 so closing down 2024-12-09T08:20:47,407 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T08:20:47,409 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:47,409 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T08:20:47,410 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:47,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741841_1017 (size=276) 2024-12-09T08:20:47,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741841_1017 (size=276) 2024-12-09T08:20:47,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741841_1017 (size=276) 2024-12-09T08:20:47,424 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T08:20:47,427 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002 2024-12-09T08:20:47,432 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-09T08:20:47,432 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1, journal: Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1, size=276 (276bytes) at 1733732447382Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1 so closing down at 1733732447403 (+21 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000001-wal-1.temp at 1733732447409 (+6 ms)3 split writer threads finished at 1733732447410 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733732447424 (+14 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002 at 1733732447427 (+3 ms)Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733732447432 (+5 ms) 2024-12-09T08:20:47,450 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2, size=230 (230bytes) 2024-12-09T08:20:47,450 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2 2024-12-09T08:20:47,451 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2 after 1ms 2024-12-09T08:20:47,455 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,456 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2 took 6ms 2024-12-09T08:20:47,458 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2 so closing down 2024-12-09T08:20:47,458 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:47,461 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T08:20:47,462 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T08:20:47,462 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:47,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741842_1018 (size=230) 2024-12-09T08:20:47,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741842_1018 (size=230) 2024-12-09T08:20:47,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741842_1018 (size=230) 2024-12-09T08:20:47,471 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T08:20:47,475 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,478 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-09T08:20:47,479 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-09T08:20:47,480 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2, journal: Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2, size=230 (230bytes) at 1733732447450Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2 so closing down at 1733732447458 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002-wal-2.temp at 1733732447462 (+4 ms)3 split writer threads finished at 1733732447462Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733732447471 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733732447479 (+8 ms) 2024-12-09T08:20:47,480 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:47,483 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:47,494 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal.1733732447483, exclude list is [], retry=0 2024-12-09T08:20:47,498 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:47,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:47,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:47,501 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal.1733732447483 2024-12-09T08:20:47,501 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:47,502 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d13634f38a8e7778980c4dd7e0b9c48, NAME => 'testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:47,502 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:47,502 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,502 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,504 INFO [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,505 INFO [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d13634f38a8e7778980c4dd7e0b9c48 columnFamilyName a 2024-12-09T08:20:47,505 DEBUG [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:47,506 INFO [StoreOpener-0d13634f38a8e7778980c4dd7e0b9c48-1 {}] regionserver.HStore(327): Store=0d13634f38a8e7778980c4dd7e0b9c48/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:47,507 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,508 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,512 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002 2024-12-09T08:20:47,516 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,522 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002 2024-12-09T08:20:47,525 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d13634f38a8e7778980c4dd7e0b9c48 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T08:20:47,582 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/.tmp/a/3c1d80671e0c45b4af222bbfe9651758 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733732447325/Put/seqid=0 2024-12-09T08:20:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741844_1020 (size=5170) 2024-12-09T08:20:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741844_1020 (size=5170) 2024-12-09T08:20:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741844_1020 (size=5170) 2024-12-09T08:20:47,596 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/.tmp/a/3c1d80671e0c45b4af222bbfe9651758 2024-12-09T08:20:47,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/.tmp/a/3c1d80671e0c45b4af222bbfe9651758 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/a/3c1d80671e0c45b4af222bbfe9651758 2024-12-09T08:20:47,644 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/a/3c1d80671e0c45b4af222bbfe9651758, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T08:20:47,652 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0d13634f38a8e7778980c4dd7e0b9c48 in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T08:20:47,654 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/0000000000000000002 2024-12-09T08:20:47,655 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,655 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,658 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d13634f38a8e7778980c4dd7e0b9c48 2024-12-09T08:20:47,662 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/0d13634f38a8e7778980c4dd7e0b9c48/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T08:20:47,664 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d13634f38a8e7778980c4dd7e0b9c48; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68755125, jitterRate=0.024531200528144836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:20:47,665 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d13634f38a8e7778980c4dd7e0b9c48: Writing region info on filesystem at 1733732447502Initializing all the Stores at 1733732447503 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732447503Obtaining lock to block concurrent updates at 1733732447525 (+22 ms)Preparing flush snapshotting stores in 0d13634f38a8e7778980c4dd7e0b9c48 at 1733732447525Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733732447529 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733732447251.0d13634f38a8e7778980c4dd7e0b9c48. at 1733732447529Flushing 0d13634f38a8e7778980c4dd7e0b9c48/a: creating writer at 1733732447530 (+1 ms)Flushing 0d13634f38a8e7778980c4dd7e0b9c48/a: appending metadata at 1733732447573 (+43 ms)Flushing 0d13634f38a8e7778980c4dd7e0b9c48/a: closing flushed file at 1733732447575 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60efa2: reopening flushed file at 1733732447633 (+58 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0d13634f38a8e7778980c4dd7e0b9c48 in 124ms, sequenceid=2, compaction requested=false; wal=null at 1733732447652 (+19 ms)Cleaning up temporary data from old regions at 1733732447655 (+3 ms)Region opened successfully at 1733732447665 (+10 ms) 2024-12-09T08:20:47,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=380 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:56988 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:37444 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:37580 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:57004 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40523/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:34952 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=4484 (was 4492) 2024-12-09T08:20:47,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=4468 2024-12-09T08:20:47,717 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:47,719 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:47,720 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:47,723 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05719189, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-05719189, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:47,735 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05719189/hregion-05719189.1733732447724, exclude list is [], retry=0 2024-12-09T08:20:47,738 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:47,738 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:47,739 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:47,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05719189/hregion-05719189.1733732447724 2024-12-09T08:20:47,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:47,746 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 690a174ab4c1aee2269d0260c43c7702, NAME => 'testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:20:47,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741846_1022 (size=64) 2024-12-09T08:20:47,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741846_1022 (size=64) 2024-12-09T08:20:47,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741846_1022 (size=64) 2024-12-09T08:20:47,757 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:47,766 INFO [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,769 INFO [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690a174ab4c1aee2269d0260c43c7702 columnFamilyName a 2024-12-09T08:20:47,770 DEBUG [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:47,771 INFO [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] regionserver.HStore(327): Store=690a174ab4c1aee2269d0260c43c7702/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:47,771 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,772 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,773 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,773 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,775 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,778 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:47,779 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690a174ab4c1aee2269d0260c43c7702; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61730413, jitterRate=-0.08014516532421112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:20:47,779 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690a174ab4c1aee2269d0260c43c7702: Writing region info on filesystem at 1733732447757Initializing all the Stores at 1733732447759 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732447759Cleaning up temporary data from old regions at 1733732447773 (+14 ms)Region opened successfully at 1733732447779 (+6 ms) 2024-12-09T08:20:47,780 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 690a174ab4c1aee2269d0260c43c7702, disabling compactions & flushes 2024-12-09T08:20:47,780 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702. 2024-12-09T08:20:47,780 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702. 2024-12-09T08:20:47,780 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702. after waiting 0 ms 2024-12-09T08:20:47,780 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702. 2024-12-09T08:20:47,780 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702. 2024-12-09T08:20:47,780 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 690a174ab4c1aee2269d0260c43c7702: Waiting for close lock at 1733732447780Disabling compacts and flushes for region at 1733732447780Disabling writes for close at 1733732447780Writing region close event to WAL at 1733732447780Closed at 1733732447780 2024-12-09T08:20:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741845_1021 (size=93) 2024-12-09T08:20:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741845_1021 (size=93) 2024-12-09T08:20:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741845_1021 (size=93) 2024-12-09T08:20:47,787 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:47,787 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05719189:(num 1733732447724) 2024-12-09T08:20:47,788 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:20:47,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741847_1023 (size=276) 2024-12-09T08:20:47,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741847_1023 (size=276) 2024-12-09T08:20:47,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741847_1023 (size=276) 2024-12-09T08:20:47,804 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:20:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741848_1024 (size=230) 2024-12-09T08:20:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741848_1024 (size=230) 2024-12-09T08:20:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741848_1024 (size=230) 2024-12-09T08:20:47,827 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2, size=230 (230bytes) 2024-12-09T08:20:47,827 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2 2024-12-09T08:20:47,828 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2 after 1ms 2024-12-09T08:20:47,831 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,831 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2 took 4ms 2024-12-09T08:20:47,834 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2 so closing down 2024-12-09T08:20:47,834 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:47,836 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T08:20:47,837 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T08:20:47,837 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:47,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741849_1025 (size=230) 2024-12-09T08:20:47,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741849_1025 (size=230) 2024-12-09T08:20:47,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741849_1025 (size=230) 2024-12-09T08:20:47,846 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T08:20:47,847 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 2024-12-09T08:20:47,848 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-09T08:20:47,848 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2, journal: Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2, size=230 (230bytes) at 1733732447827Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2 so closing down at 1733732447834 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002-wal-2.temp at 1733732447837 (+3 ms)3 split writer threads finished at 1733732447838 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733732447846 (+8 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 at 1733732447847 (+1 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733732447848 (+1 ms) 2024-12-09T08:20:47,860 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1, size=276 (276bytes) 2024-12-09T08:20:47,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1 2024-12-09T08:20:47,861 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1 after 1ms 2024-12-09T08:20:47,864 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,864 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1 took 4ms 2024-12-09T08:20:47,867 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1 so closing down 2024-12-09T08:20:47,867 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:47,870 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T08:20:47,871 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T08:20:47,872 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:47,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741850_1026 (size=276) 2024-12-09T08:20:47,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741850_1026 (size=276) 2024-12-09T08:20:47,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741850_1026 (size=276) 2024-12-09T08:20:47,880 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T08:20:47,883 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,885 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002, length=230 2024-12-09T08:20:47,887 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 2024-12-09T08:20:47,887 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-09T08:20:47,887 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1, journal: Splitting hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1, size=276 (276bytes) at 1733732447860Finishing writing output for hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1 so closing down at 1733732447867 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000001-wal-1.temp at 1733732447871 (+4 ms)3 split writer threads finished at 1733732447872 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733732447880 (+8 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 at 1733732447887 (+7 ms)Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733732447887 2024-12-09T08:20:47,888 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:47,890 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:47,901 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal.1733732447891, exclude list is [], retry=0 2024-12-09T08:20:47,905 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:47,905 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:47,906 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:47,908 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal.1733732447891 2024-12-09T08:20:47,909 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:47,909 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 690a174ab4c1aee2269d0260c43c7702, NAME => 'testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:47,909 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:47,909 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,909 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,912 INFO [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,914 INFO [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690a174ab4c1aee2269d0260c43c7702 columnFamilyName a 2024-12-09T08:20:47,914 DEBUG [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:47,915 INFO [StoreOpener-690a174ab4c1aee2269d0260c43c7702-1 {}] regionserver.HStore(327): Store=690a174ab4c1aee2269d0260c43c7702/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:47,915 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,916 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,918 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,919 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 2024-12-09T08:20:47,922 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:47,924 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 2024-12-09T08:20:47,924 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 690a174ab4c1aee2269d0260c43c7702 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T08:20:47,940 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/.tmp/a/f9af73e0da834a06a2799442e911f905 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733732447787/Put/seqid=0 2024-12-09T08:20:47,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741852_1028 (size=5170) 2024-12-09T08:20:47,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741852_1028 (size=5170) 2024-12-09T08:20:47,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741852_1028 (size=5170) 2024-12-09T08:20:47,948 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/.tmp/a/f9af73e0da834a06a2799442e911f905 2024-12-09T08:20:47,956 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/.tmp/a/f9af73e0da834a06a2799442e911f905 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/a/f9af73e0da834a06a2799442e911f905 2024-12-09T08:20:47,964 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/a/f9af73e0da834a06a2799442e911f905, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T08:20:47,964 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 690a174ab4c1aee2269d0260c43c7702 in 40ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T08:20:47,965 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/0000000000000000002 2024-12-09T08:20:47,966 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,966 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,968 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690a174ab4c1aee2269d0260c43c7702 2024-12-09T08:20:47,972 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/690a174ab4c1aee2269d0260c43c7702/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T08:20:47,974 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690a174ab4c1aee2269d0260c43c7702; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75326909, jitterRate=0.12245841324329376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:20:47,974 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690a174ab4c1aee2269d0260c43c7702: Writing region info on filesystem at 1733732447909Initializing all the Stores at 1733732447911 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732447911Obtaining lock to block concurrent updates at 1733732447924 (+13 ms)Preparing flush snapshotting stores in 690a174ab4c1aee2269d0260c43c7702 at 1733732447924Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733732447924Flushing stores of testReplayEditsWrittenIntoWAL,,1733732447717.690a174ab4c1aee2269d0260c43c7702. at 1733732447924Flushing 690a174ab4c1aee2269d0260c43c7702/a: creating writer at 1733732447924Flushing 690a174ab4c1aee2269d0260c43c7702/a: appending metadata at 1733732447938 (+14 ms)Flushing 690a174ab4c1aee2269d0260c43c7702/a: closing flushed file at 1733732447938Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b6f5f6c: reopening flushed file at 1733732447955 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 690a174ab4c1aee2269d0260c43c7702 in 40ms, sequenceid=2, compaction requested=false; wal=null at 1733732447964 (+9 ms)Cleaning up temporary data from old regions at 1733732447966 (+2 ms)Region opened successfully at 1733732447974 (+8 ms) 2024-12-09T08:20:47,990 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:34824 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:57074 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:35022 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:37444 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:37660 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:57004 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=4459 (was 4468) 2024-12-09T08:20:48,001 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=4459 2024-12-09T08:20:48,013 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:48,015 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:48,016 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:48,019 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23639879, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-23639879, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:48,030 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23639879/hregion-23639879.1733732448020, exclude list is [], retry=0 2024-12-09T08:20:48,034 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:48,034 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:48,034 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:48,036 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23639879/hregion-23639879.1733732448020 2024-12-09T08:20:48,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:48,037 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dab858fbcc02f86c509a9d2cb155e1e3, NAME => 'testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:20:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741854_1030 (size=64) 2024-12-09T08:20:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741854_1030 (size=64) 2024-12-09T08:20:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741854_1030 (size=64) 2024-12-09T08:20:48,048 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:48,050 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,052 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dab858fbcc02f86c509a9d2cb155e1e3 columnFamilyName a 2024-12-09T08:20:48,052 DEBUG [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:48,052 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(327): Store=dab858fbcc02f86c509a9d2cb155e1e3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:48,053 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,054 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dab858fbcc02f86c509a9d2cb155e1e3 columnFamilyName b 2024-12-09T08:20:48,054 DEBUG [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:48,055 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(327): Store=dab858fbcc02f86c509a9d2cb155e1e3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:48,055 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,057 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dab858fbcc02f86c509a9d2cb155e1e3 columnFamilyName c 2024-12-09T08:20:48,057 DEBUG [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:48,058 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(327): Store=dab858fbcc02f86c509a9d2cb155e1e3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:48,058 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,058 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,059 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,060 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,060 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,061 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:20:48,062 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:48,066 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:48,067 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dab858fbcc02f86c509a9d2cb155e1e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73040440, jitterRate=0.0883873701095581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:20:48,067 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dab858fbcc02f86c509a9d2cb155e1e3: Writing region info on filesystem at 1733732448048Initializing all the Stores at 1733732448049 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732448049Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732448050 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732448050Cleaning up temporary data from old regions at 1733732448060 (+10 ms)Region opened successfully at 1733732448067 (+7 ms) 2024-12-09T08:20:48,067 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dab858fbcc02f86c509a9d2cb155e1e3, disabling compactions & flushes 2024-12-09T08:20:48,067 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:48,067 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:48,067 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. after waiting 0 ms 2024-12-09T08:20:48,067 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:48,068 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:48,068 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dab858fbcc02f86c509a9d2cb155e1e3: Waiting for close lock at 1733732448067Disabling compacts and flushes for region at 1733732448067Disabling writes for close at 1733732448067Writing region close event to WAL at 1733732448068 (+1 ms)Closed at 1733732448068 2024-12-09T08:20:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741853_1029 (size=93) 2024-12-09T08:20:48,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741853_1029 (size=93) 2024-12-09T08:20:48,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741853_1029 (size=93) 2024-12-09T08:20:48,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:48,077 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23639879:(num 1733732448020) 2024-12-09T08:20:48,077 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:48,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:48,096 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080, exclude list is [], retry=0 2024-12-09T08:20:48,099 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:48,100 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:48,100 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:48,102 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 2024-12-09T08:20:48,102 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:48,404 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080, size=0 (0bytes) 2024-12-09T08:20:48,404 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 might be still open, length is 0 2024-12-09T08:20:48,404 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 2024-12-09T08:20:48,405 WARN [IPC Server handler 2 on default port 40523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T08:20:48,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 after 2ms 2024-12-09T08:20:48,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:37690 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41795:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37690 dst: /127.0.0.1:41795 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41795 remote=/127.0.0.1:37690]. Total timeout mills is 60000, 59493 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:48,867 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:57106 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38773:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57106 dst: /127.0.0.1:38773 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:48,867 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:35046 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35046 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:48,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741855_1032 (size=200589) 2024-12-09T08:20:48,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741855_1032 (size=200589) 2024-12-09T08:20:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741855_1032 (size=200589) 2024-12-09T08:20:52,306 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:20:52,348 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T08:20:52,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 after 4003ms 2024-12-09T08:20:52,410 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:52,412 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 took 4009ms 2024-12-09T08:20:52,416 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733732448080.temp 2024-12-09T08:20:52,421 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000000001-wal.1733732448080.temp 2024-12-09T08:20:52,510 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080; continuing. 2024-12-09T08:20:52,511 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 so closing down 2024-12-09T08:20:52,511 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:52,511 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:52,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741856_1033 (size=200597) 2024-12-09T08:20:52,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741856_1033 (size=200597) 2024-12-09T08:20:52,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741856_1033 (size=200597) 2024-12-09T08:20:52,526 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000000001-wal.1733732448080.temp (wrote 3002 edits, skipped 0 edits in 46 ms) 2024-12-09T08:20:52,528 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000000001-wal.1733732448080.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002 2024-12-09T08:20:52,528 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:20:52,529 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080, journal: Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080, size=0 (0bytes) at 1733732448404Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000000001-wal.1733732448080.temp at 1733732452421 (+4017 ms)Split 1024 edits, skipped 0 edits. at 1733732452469 (+48 ms)Split 2048 edits, skipped 0 edits. at 1733732452489 (+20 ms)Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 so closing down at 1733732452511 (+22 ms)3 split writer threads finished at 1733732452511Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000000001-wal.1733732448080.temp (wrote 3002 edits, skipped 0 edits in 46 ms) at 1733732452526 (+15 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000000001-wal.1733732448080.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002 at 1733732452528 (+2 ms)Processed 3002 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080, size=0, length=0, corrupted=false, cancelled=false at 1733732452528 2024-12-09T08:20:52,531 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732448080 2024-12-09T08:20:52,533 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002 2024-12-09T08:20:52,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:52,535 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:52,548 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732452536, exclude list is [], retry=0 2024-12-09T08:20:52,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:52,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:52,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:52,557 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732452536 2024-12-09T08:20:52,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:20:52,558 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:52,561 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:52,562 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dab858fbcc02f86c509a9d2cb155e1e3 columnFamilyName a 2024-12-09T08:20:52,562 DEBUG [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:52,564 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(327): Store=dab858fbcc02f86c509a9d2cb155e1e3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:52,564 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:52,566 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dab858fbcc02f86c509a9d2cb155e1e3 columnFamilyName b 2024-12-09T08:20:52,566 DEBUG [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:52,567 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(327): Store=dab858fbcc02f86c509a9d2cb155e1e3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:52,567 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:52,569 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dab858fbcc02f86c509a9d2cb155e1e3 columnFamilyName c 2024-12-09T08:20:52,569 DEBUG [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:52,570 INFO [StoreOpener-dab858fbcc02f86c509a9d2cb155e1e3-1 {}] regionserver.HStore(327): Store=dab858fbcc02f86c509a9d2cb155e1e3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:52,570 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:52,571 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:52,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:52,577 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002 2024-12-09T08:20:52,581 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:52,629 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:20:52,987 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dab858fbcc02f86c509a9d2cb155e1e3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T08:20:53,022 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/a/eb25421b96044c918d77db2463e0727b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733732448114/Put/seqid=0 2024-12-09T08:20:53,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741858_1035 (size=50463) 2024-12-09T08:20:53,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741858_1035 (size=50463) 2024-12-09T08:20:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741858_1035 (size=50463) 2024-12-09T08:20:53,037 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/a/eb25421b96044c918d77db2463e0727b 2024-12-09T08:20:53,045 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/a/eb25421b96044c918d77db2463e0727b as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/a/eb25421b96044c918d77db2463e0727b 2024-12-09T08:20:53,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/a/eb25421b96044c918d77db2463e0727b, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T08:20:53,053 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for dab858fbcc02f86c509a9d2cb155e1e3 in 66ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T08:20:53,081 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:20:53,081 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dab858fbcc02f86c509a9d2cb155e1e3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T08:20:53,091 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/a/341f748d349c40e39920bdbdaaaacd17 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733732448179/Put/seqid=0 2024-12-09T08:20:53,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741859_1036 (size=20072) 2024-12-09T08:20:53,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741859_1036 (size=20072) 2024-12-09T08:20:53,113 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/a/341f748d349c40e39920bdbdaaaacd17 2024-12-09T08:20:53,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741859_1036 (size=20072) 2024-12-09T08:20:53,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/b/f8ff0fb6f541471fb34ad82aed6a9e1e is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733732448206/Put/seqid=0 2024-12-09T08:20:53,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741860_1037 (size=35835) 2024-12-09T08:20:53,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741860_1037 (size=35835) 2024-12-09T08:20:53,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741860_1037 (size=35835) 2024-12-09T08:20:53,161 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/b/f8ff0fb6f541471fb34ad82aed6a9e1e 2024-12-09T08:20:53,170 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/a/341f748d349c40e39920bdbdaaaacd17 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/a/341f748d349c40e39920bdbdaaaacd17 2024-12-09T08:20:53,180 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/a/341f748d349c40e39920bdbdaaaacd17, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T08:20:53,181 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/b/f8ff0fb6f541471fb34ad82aed6a9e1e as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/b/f8ff0fb6f541471fb34ad82aed6a9e1e 2024-12-09T08:20:53,189 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/b/f8ff0fb6f541471fb34ad82aed6a9e1e, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T08:20:53,189 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for dab858fbcc02f86c509a9d2cb155e1e3 in 108ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T08:20:53,206 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:20:53,206 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dab858fbcc02f86c509a9d2cb155e1e3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T08:20:53,215 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/b/242d4fdc9fc04822aa1404c74a3928bb is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733732448234/Put/seqid=0 2024-12-09T08:20:53,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741861_1038 (size=35082) 2024-12-09T08:20:53,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741861_1038 (size=35082) 2024-12-09T08:20:53,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741861_1038 (size=35082) 2024-12-09T08:20:53,229 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/b/242d4fdc9fc04822aa1404c74a3928bb 2024-12-09T08:20:53,262 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/c/6121898291e6410db4dd3d90b0279e8e is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733732448284/Put/seqid=0 2024-12-09T08:20:53,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741862_1039 (size=20825) 2024-12-09T08:20:53,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741862_1039 (size=20825) 2024-12-09T08:20:53,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741862_1039 (size=20825) 2024-12-09T08:20:53,275 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/c/6121898291e6410db4dd3d90b0279e8e 2024-12-09T08:20:53,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/b/242d4fdc9fc04822aa1404c74a3928bb as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/b/242d4fdc9fc04822aa1404c74a3928bb 2024-12-09T08:20:53,291 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/b/242d4fdc9fc04822aa1404c74a3928bb, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T08:20:53,293 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/c/6121898291e6410db4dd3d90b0279e8e as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/c/6121898291e6410db4dd3d90b0279e8e 2024-12-09T08:20:53,299 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/c/6121898291e6410db4dd3d90b0279e8e, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T08:20:53,300 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for dab858fbcc02f86c509a9d2cb155e1e3 in 93ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T08:20:53,309 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733732448357/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:53,313 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002 2024-12-09T08:20:53,313 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:20:53,313 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dab858fbcc02f86c509a9d2cb155e1e3 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T08:20:53,320 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/c/82415747526241329d04a45886f12073 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733732448297/Put/seqid=0 2024-12-09T08:20:53,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741863_1040 (size=50301) 2024-12-09T08:20:53,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741863_1040 (size=50301) 2024-12-09T08:20:53,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741863_1040 (size=50301) 2024-12-09T08:20:53,333 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/c/82415747526241329d04a45886f12073 2024-12-09T08:20:53,341 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 82415747526241329d04a45886f12073 2024-12-09T08:20:53,343 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/.tmp/c/82415747526241329d04a45886f12073 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/c/82415747526241329d04a45886f12073 2024-12-09T08:20:53,351 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 82415747526241329d04a45886f12073 2024-12-09T08:20:53,351 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/c/82415747526241329d04a45886f12073, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T08:20:53,352 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for dab858fbcc02f86c509a9d2cb155e1e3 in 39ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T08:20:53,353 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/0000000000000003002 2024-12-09T08:20:53,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:53,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:53,356 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:20:53,358 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dab858fbcc02f86c509a9d2cb155e1e3 2024-12-09T08:20:53,361 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenIntoWAL/dab858fbcc02f86c509a9d2cb155e1e3/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T08:20:53,362 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dab858fbcc02f86c509a9d2cb155e1e3; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61239203, jitterRate=-0.08746476471424103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:20:53,362 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dab858fbcc02f86c509a9d2cb155e1e3: Writing region info on filesystem at 1733732452558Initializing all the Stores at 1733732452559 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732452559Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732452561 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732452561Cleaning up temporary data from old regions at 1733732453355 (+794 ms)Region opened successfully at 1733732453362 (+7 ms) 2024-12-09T08:20:53,431 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dab858fbcc02f86c509a9d2cb155e1e3, disabling compactions & flushes 2024-12-09T08:20:53,431 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:53,431 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:53,431 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. after waiting 0 ms 2024-12-09T08:20:53,431 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:53,434 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732448014.dab858fbcc02f86c509a9d2cb155e1e3. 2024-12-09T08:20:53,434 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dab858fbcc02f86c509a9d2cb155e1e3: Waiting for close lock at 1733732453431Disabling compacts and flushes for region at 1733732453431Disabling writes for close at 1733732453431Writing region close event to WAL at 1733732453433 (+2 ms)Closed at 1733732453434 (+1 ms) 2024-12-09T08:20:53,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741857_1034 (size=93) 2024-12-09T08:20:53,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741857_1034 (size=93) 2024-12-09T08:20:53,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741857_1034 (size=93) 2024-12-09T08:20:53,442 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:53,442 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732452536) 2024-12-09T08:20:53,457 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43985 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:43249 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076590320_22 at /127.0.0.1:55806 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40523 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:43985 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40523 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076590320_22 at /127.0.0.1:53850 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43249 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076590320_22 at /127.0.0.1:34576 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1076590320_22 at /127.0.0.1:34560 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=4397 (was 4459) 2024-12-09T08:20:53,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=4396 2024-12-09T08:20:53,484 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:53,486 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:53,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:53,491 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77614872, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-77614872, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:53,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77614872/hregion-77614872.1733732453491, exclude list is [], retry=0 2024-12-09T08:20:53,507 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:53,507 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:53,508 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:53,510 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77614872/hregion-77614872.1733732453491 2024-12-09T08:20:53,511 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:53,511 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 01039496ac3e5f43d2ca1890d0e66c29, NAME => 'test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:20:53,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741865_1042 (size=43) 2024-12-09T08:20:53,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741865_1042 (size=43) 2024-12-09T08:20:53,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741865_1042 (size=43) 2024-12-09T08:20:53,521 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:53,523 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,525 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01039496ac3e5f43d2ca1890d0e66c29 columnFamilyName a 2024-12-09T08:20:53,525 DEBUG [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:53,525 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(327): Store=01039496ac3e5f43d2ca1890d0e66c29/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:53,526 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,527 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01039496ac3e5f43d2ca1890d0e66c29 columnFamilyName b 2024-12-09T08:20:53,527 DEBUG [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:53,528 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(327): Store=01039496ac3e5f43d2ca1890d0e66c29/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:53,528 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,530 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01039496ac3e5f43d2ca1890d0e66c29 columnFamilyName c 2024-12-09T08:20:53,530 DEBUG [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:53,530 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(327): Store=01039496ac3e5f43d2ca1890d0e66c29/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:53,531 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,532 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,533 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,533 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,534 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:20:53,536 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:53,538 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:53,539 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 01039496ac3e5f43d2ca1890d0e66c29; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65756450, jitterRate=-0.020152539014816284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:20:53,540 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 01039496ac3e5f43d2ca1890d0e66c29: Writing region info on filesystem at 1733732453521Initializing all the Stores at 1733732453522 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732453522Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732453523 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732453523Cleaning up temporary data from old regions at 1733732453533 (+10 ms)Region opened successfully at 1733732453540 (+7 ms) 2024-12-09T08:20:53,540 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 01039496ac3e5f43d2ca1890d0e66c29, disabling compactions & flushes 2024-12-09T08:20:53,540 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:53,540 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:53,540 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. after waiting 0 ms 2024-12-09T08:20:53,540 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:53,541 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:53,541 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 01039496ac3e5f43d2ca1890d0e66c29: Waiting for close lock at 1733732453540Disabling compacts and flushes for region at 1733732453540Disabling writes for close at 1733732453540Writing region close event to WAL at 1733732453541 (+1 ms)Closed at 1733732453541 2024-12-09T08:20:53,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741864_1041 (size=93) 2024-12-09T08:20:53,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741864_1041 (size=93) 2024-12-09T08:20:53,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741864_1041 (size=93) 2024-12-09T08:20:53,546 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:53,546 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77614872:(num 1733732453491) 2024-12-09T08:20:53,547 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:53,549 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:53,561 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549, exclude list is [], retry=0 2024-12-09T08:20:53,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:53,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:53,565 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:53,567 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 2024-12-09T08:20:53,568 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:53,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741866_1043 (size=200357) 2024-12-09T08:20:53,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741866_1043 (size=200357) 2024-12-09T08:20:53,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741866_1043 (size=200357) 2024-12-09T08:20:53,754 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549, size=195.7 K (200357bytes) 2024-12-09T08:20:53,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 2024-12-09T08:20:53,755 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 after 1ms 2024-12-09T08:20:53,758 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:53,760 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 took 6ms 2024-12-09T08:20:53,764 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733732453549.temp 2024-12-09T08:20:53,771 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000000001-wal.1733732453549.temp 2024-12-09T08:20:53,818 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 so closing down 2024-12-09T08:20:53,818 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:53,818 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:53,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741867_1044 (size=200357) 2024-12-09T08:20:53,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741867_1044 (size=200357) 2024-12-09T08:20:53,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741867_1044 (size=200357) 2024-12-09T08:20:53,829 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000000001-wal.1733732453549.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-12-09T08:20:53,830 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000000001-wal.1733732453549.temp to hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000 2024-12-09T08:20:53,831 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-09T08:20:53,831 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549, journal: Splitting hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549, size=195.7 K (200357bytes) at 1733732453754Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000000001-wal.1733732453549.temp at 1733732453771 (+17 ms)Split 1024 edits, skipped 0 edits. at 1733732453781 (+10 ms)Split 2048 edits, skipped 0 edits. at 1733732453799 (+18 ms)Finishing writing output for hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 so closing down at 1733732453818 (+19 ms)3 split writer threads finished at 1733732453818Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000000001-wal.1733732453549.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1733732453829 (+11 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000000001-wal.1733732453549.temp to hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000 at 1733732453831 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733732453831 2024-12-09T08:20:53,833 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453549 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732453549 2024-12-09T08:20:53,834 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000 2024-12-09T08:20:53,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:53,836 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:53,849 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837, exclude list is [], retry=0 2024-12-09T08:20:53,852 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:53,853 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:53,853 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:53,855 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 2024-12-09T08:20:53,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:20:53,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741868_1045 (size=200484) 2024-12-09T08:20:53,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741868_1045 (size=200484) 2024-12-09T08:20:53,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741868_1045 (size=200484) 2024-12-09T08:20:54,013 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837, size=195.8 K (200484bytes) 2024-12-09T08:20:54,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 2024-12-09T08:20:54,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 after 0ms 2024-12-09T08:20:54,016 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:54,018 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 took 4ms 2024-12-09T08:20:54,022 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733732453837.temp 2024-12-09T08:20:54,024 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003001-wal.1733732453837.temp 2024-12-09T08:20:54,074 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 so closing down 2024-12-09T08:20:54,074 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:54,074 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741869_1046 (size=200484) 2024-12-09T08:20:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741869_1046 (size=200484) 2024-12-09T08:20:54,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741869_1046 (size=200484) 2024-12-09T08:20:54,080 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003001-wal.1733732453837.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-12-09T08:20:54,081 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003001-wal.1733732453837.temp to hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000 2024-12-09T08:20:54,081 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 63 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-09T08:20:54,081 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837, journal: Splitting hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837, size=195.8 K (200484bytes) at 1733732454013Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003001-wal.1733732453837.temp at 1733732454024 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733732454038 (+14 ms)Split 2048 edits, skipped 0 edits. at 1733732454056 (+18 ms)Finishing writing output for hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 so closing down at 1733732454074 (+18 ms)3 split writer threads finished at 1733732454074Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003001-wal.1733732453837.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1733732454080 (+6 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003001-wal.1733732453837.temp to hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000 at 1733732454081 (+1 ms)Processed 3000 edits across 1 Regions in 63 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733732454081 2024-12-09T08:20:54,083 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732453837 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732453837 2024-12-09T08:20:54,085 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000 2024-12-09T08:20:54,085 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:54,087 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/test2727-manual,16010,1733732453484, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:54,089 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T08:20:54,089 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T08:20:54,091 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T08:20:54,091 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T08:20:54,092 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T08:20:54,092 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T08:20:54,092 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T08:20:54,093 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T08:20:54,101 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732454087, exclude list is [], retry=0 2024-12-09T08:20:54,104 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:54,104 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:54,104 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:54,119 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733732453484/wal.1733732454087 2024-12-09T08:20:54,119 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:20:54,119 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 01039496ac3e5f43d2ca1890d0e66c29, NAME => 'test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:54,120 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:54,120 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,120 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,137 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,138 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01039496ac3e5f43d2ca1890d0e66c29 columnFamilyName a 2024-12-09T08:20:54,138 DEBUG [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:54,139 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(327): Store=01039496ac3e5f43d2ca1890d0e66c29/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:54,139 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,140 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01039496ac3e5f43d2ca1890d0e66c29 columnFamilyName b 2024-12-09T08:20:54,140 DEBUG [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:54,141 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(327): Store=01039496ac3e5f43d2ca1890d0e66c29/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:54,141 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,142 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01039496ac3e5f43d2ca1890d0e66c29 columnFamilyName c 2024-12-09T08:20:54,142 DEBUG [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:54,143 INFO [StoreOpener-01039496ac3e5f43d2ca1890d0e66c29-1 {}] regionserver.HStore(327): Store=01039496ac3e5f43d2ca1890d0e66c29/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:54,143 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,144 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,146 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,146 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000 2024-12-09T08:20:54,149 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:54,192 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000 2024-12-09T08:20:54,194 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000 2024-12-09T08:20:54,197 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:54,263 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000 2024-12-09T08:20:54,264 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 01039496ac3e5f43d2ca1890d0e66c29 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T08:20:54,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/a/24882a21d90b43f9aaddc9314f6034f0 is 41, key is test2727/a:100/1733732453860/Put/seqid=0 2024-12-09T08:20:54,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741871_1048 (size=84227) 2024-12-09T08:20:54,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741871_1048 (size=84227) 2024-12-09T08:20:54,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741871_1048 (size=84227) 2024-12-09T08:20:54,306 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/a/24882a21d90b43f9aaddc9314f6034f0 2024-12-09T08:20:54,331 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/b/dd8fd671cdb94b4481da287168542a90 is 41, key is test2727/b:100/1733732453903/Put/seqid=0 2024-12-09T08:20:54,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741872_1049 (size=84609) 2024-12-09T08:20:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741872_1049 (size=84609) 2024-12-09T08:20:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741872_1049 (size=84609) 2024-12-09T08:20:54,744 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/b/dd8fd671cdb94b4481da287168542a90 2024-12-09T08:20:54,776 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/c/eb6bdd9ffc3d439cb68c3d67858fc411 is 41, key is test2727/c:100/1733732453948/Put/seqid=0 2024-12-09T08:20:54,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741873_1050 (size=84609) 2024-12-09T08:20:54,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741873_1050 (size=84609) 2024-12-09T08:20:54,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741873_1050 (size=84609) 2024-12-09T08:20:54,785 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/c/eb6bdd9ffc3d439cb68c3d67858fc411 2024-12-09T08:20:54,792 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/a/24882a21d90b43f9aaddc9314f6034f0 as hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/a/24882a21d90b43f9aaddc9314f6034f0 2024-12-09T08:20:54,799 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/a/24882a21d90b43f9aaddc9314f6034f0, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T08:20:54,800 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/b/dd8fd671cdb94b4481da287168542a90 as hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/b/dd8fd671cdb94b4481da287168542a90 2024-12-09T08:20:54,811 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/b/dd8fd671cdb94b4481da287168542a90, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T08:20:54,813 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/.tmp/c/eb6bdd9ffc3d439cb68c3d67858fc411 as hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/c/eb6bdd9ffc3d439cb68c3d67858fc411 2024-12-09T08:20:54,819 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/c/eb6bdd9ffc3d439cb68c3d67858fc411, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T08:20:54,820 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 01039496ac3e5f43d2ca1890d0e66c29 in 556ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T08:20:54,821 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000003000 2024-12-09T08:20:54,822 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/0000000000000006000 2024-12-09T08:20:54,823 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,824 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,824 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:20:54,826 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 01039496ac3e5f43d2ca1890d0e66c29 2024-12-09T08:20:54,829 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/test2727/01039496ac3e5f43d2ca1890d0e66c29/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T08:20:54,831 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 01039496ac3e5f43d2ca1890d0e66c29; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67737265, jitterRate=0.009363904595375061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:20:54,832 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 01039496ac3e5f43d2ca1890d0e66c29: Writing region info on filesystem at 1733732454120Initializing all the Stores at 1733732454121 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732454121Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732454137 (+16 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732454137Obtaining lock to block concurrent updates at 1733732454264 (+127 ms)Preparing flush snapshotting stores in 01039496ac3e5f43d2ca1890d0e66c29 at 1733732454264Finished memstore snapshotting test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733732454264Flushing stores of test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. at 1733732454264Flushing 01039496ac3e5f43d2ca1890d0e66c29/a: creating writer at 1733732454264Flushing 01039496ac3e5f43d2ca1890d0e66c29/a: appending metadata at 1733732454292 (+28 ms)Flushing 01039496ac3e5f43d2ca1890d0e66c29/a: closing flushed file at 1733732454292Flushing 01039496ac3e5f43d2ca1890d0e66c29/b: creating writer at 1733732454312 (+20 ms)Flushing 01039496ac3e5f43d2ca1890d0e66c29/b: appending metadata at 1733732454329 (+17 ms)Flushing 01039496ac3e5f43d2ca1890d0e66c29/b: closing flushed file at 1733732454330 (+1 ms)Flushing 01039496ac3e5f43d2ca1890d0e66c29/c: creating writer at 1733732454751 (+421 ms)Flushing 01039496ac3e5f43d2ca1890d0e66c29/c: appending metadata at 1733732454775 (+24 ms)Flushing 01039496ac3e5f43d2ca1890d0e66c29/c: closing flushed file at 1733732454775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33e82289: reopening flushed file at 1733732454790 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3836c64e: reopening flushed file at 1733732454799 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40353c4a: reopening flushed file at 1733732454811 (+12 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 01039496ac3e5f43d2ca1890d0e66c29 in 556ms, sequenceid=6000, compaction requested=false; wal=null at 1733732454820 (+9 ms)Cleaning up temporary data from old regions at 1733732454824 (+4 ms)Region opened successfully at 1733732454832 (+8 ms) 2024-12-09T08:20:54,833 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T08:20:54,834 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 01039496ac3e5f43d2ca1890d0e66c29, disabling compactions & flushes 2024-12-09T08:20:54,834 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:54,834 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:54,834 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. after waiting 0 ms 2024-12-09T08:20:54,834 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:54,835 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733732453485.01039496ac3e5f43d2ca1890d0e66c29. 2024-12-09T08:20:54,835 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 01039496ac3e5f43d2ca1890d0e66c29: Waiting for close lock at 1733732454834Disabling compacts and flushes for region at 1733732454834Disabling writes for close at 1733732454834Writing region close event to WAL at 1733732454835 (+1 ms)Closed at 1733732454835 2024-12-09T08:20:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741870_1047 (size=93) 2024-12-09T08:20:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741870_1047 (size=93) 2024-12-09T08:20:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741870_1047 (size=93) 2024-12-09T08:20:54,841 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:54,841 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732454087) 2024-12-09T08:20:54,855 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=406 (was 404) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:53850 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:55960 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:34714 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=252 (was 230) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4189 (was 4396) 2024-12-09T08:20:54,868 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=252, ProcessCount=11, AvailableMemoryMB=4187 2024-12-09T08:20:54,881 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:54,887 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:54,887 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732454887 2024-12-09T08:20:54,896 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887 2024-12-09T08:20:54,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:54,900 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4bb4664061453fe6bcc3753d000471a5, NAME => 'testSequentialEditLogSeqNum,,1733732454881.4bb4664061453fe6bcc3753d000471a5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:54,900 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733732454881.4bb4664061453fe6bcc3753d000471a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:54,900 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,900 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,901 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5 doesn't exist for region: 4bb4664061453fe6bcc3753d000471a5 on table testSequentialEditLogSeqNum 2024-12-09T08:20:54,902 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 4bb4664061453fe6bcc3753d000471a5 on table testSequentialEditLogSeqNum 2024-12-09T08:20:54,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741875_1052 (size=62) 2024-12-09T08:20:54,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741875_1052 (size=62) 2024-12-09T08:20:54,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741875_1052 (size=62) 2024-12-09T08:20:54,925 INFO [StoreOpener-4bb4664061453fe6bcc3753d000471a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,927 INFO [StoreOpener-4bb4664061453fe6bcc3753d000471a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bb4664061453fe6bcc3753d000471a5 columnFamilyName a 2024-12-09T08:20:54,927 DEBUG [StoreOpener-4bb4664061453fe6bcc3753d000471a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:54,928 INFO [StoreOpener-4bb4664061453fe6bcc3753d000471a5-1 {}] regionserver.HStore(327): Store=4bb4664061453fe6bcc3753d000471a5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:54,928 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,929 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,930 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,930 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,930 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,932 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4bb4664061453fe6bcc3753d000471a5 2024-12-09T08:20:54,934 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:54,935 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4bb4664061453fe6bcc3753d000471a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61411413, jitterRate=-0.08489863574504852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:20:54,936 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4bb4664061453fe6bcc3753d000471a5: Writing region info on filesystem at 1733732454900Initializing all the Stores at 1733732454925 (+25 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732454925Cleaning up temporary data from old regions at 1733732454930 (+5 ms)Region opened successfully at 1733732454936 (+6 ms) 2024-12-09T08:20:54,951 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4bb4664061453fe6bcc3753d000471a5 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T08:20:54,972 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/.tmp/a/23ba61b80a45460188928aaaedd20d34 is 81, key is testSequentialEditLogSeqNum/a:x0/1733732454936/Put/seqid=0 2024-12-09T08:20:54,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741876_1053 (size=5833) 2024-12-09T08:20:54,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741876_1053 (size=5833) 2024-12-09T08:20:54,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741876_1053 (size=5833) 2024-12-09T08:20:54,979 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/.tmp/a/23ba61b80a45460188928aaaedd20d34 2024-12-09T08:20:54,986 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/.tmp/a/23ba61b80a45460188928aaaedd20d34 as hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/a/23ba61b80a45460188928aaaedd20d34 2024-12-09T08:20:54,992 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/a/23ba61b80a45460188928aaaedd20d34, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T08:20:54,994 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 4bb4664061453fe6bcc3753d000471a5 in 43ms, sequenceid=13, compaction requested=false 2024-12-09T08:20:54,994 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4bb4664061453fe6bcc3753d000471a5: 2024-12-09T08:20:55,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:20:55,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:20:55,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:20:55,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:20:55,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:20:55,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741874_1051 (size=1616) 2024-12-09T08:20:55,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741874_1051 (size=1616) 2024-12-09T08:20:55,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741874_1051 (size=1616) 2024-12-09T08:20:55,029 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887, size=1.6 K (1616bytes) 2024-12-09T08:20:55,029 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887 2024-12-09T08:20:55,030 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887 after 1ms 2024-12-09T08:20:55,033 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:55,034 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887 took 5ms 2024-12-09T08:20:55,038 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887 so closing down 2024-12-09T08:20:55,038 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:55,039 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732454887.temp 2024-12-09T08:20:55,041 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000003-wal.1733732454887.temp 2024-12-09T08:20:55,041 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:55,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741877_1054 (size=1175) 2024-12-09T08:20:55,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741877_1054 (size=1175) 2024-12-09T08:20:55,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741877_1054 (size=1175) 2024-12-09T08:20:55,057 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000003-wal.1733732454887.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T08:20:55,059 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000003-wal.1733732454887.temp to hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000020 2024-12-09T08:20:55,059 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-09T08:20:55,059 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887, journal: Splitting hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887, size=1.6 K (1616bytes) at 1733732455029Finishing writing output for hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887 so closing down at 1733732455038 (+9 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000003-wal.1733732454887.temp at 1733732455041 (+3 ms)3 split writer threads finished at 1733732455041Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000003-wal.1733732454887.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733732455057 (+16 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000003-wal.1733732454887.temp to hdfs://localhost:40523/hbase/data/default/testSequentialEditLogSeqNum/4bb4664061453fe6bcc3753d000471a5/recovered.edits/0000000000000000020 at 1733732455059 (+2 ms)Processed 17 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:40523/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732454880/wal.1733732454887, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733732455059 2024-12-09T08:20:55,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=413 (was 406) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:55960 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=252 (was 252), ProcessCount=11 (was 11), AvailableMemoryMB=4165 (was 4187) 2024-12-09T08:20:55,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=252, ProcessCount=11, AvailableMemoryMB=4169 2024-12-09T08:20:55,099 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:55,101 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:55,121 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:20:55,124 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70145694, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-70145694, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:55,137 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70145694/hregion-70145694.1733732455125, exclude list is [], retry=0 2024-12-09T08:20:55,140 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:55,140 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:55,140 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:55,172 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70145694/hregion-70145694.1733732455125 2024-12-09T08:20:55,172 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:55,173 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c52269ac3ef6cfb97100e50b1f62738b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:20:55,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741879_1056 (size=70) 2024-12-09T08:20:55,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741879_1056 (size=70) 2024-12-09T08:20:55,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741879_1056 (size=70) 2024-12-09T08:20:55,187 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:55,191 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,193 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName a 2024-12-09T08:20:55,193 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:55,194 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:55,194 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,196 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName b 2024-12-09T08:20:55,196 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:55,197 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:55,197 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,199 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName c 2024-12-09T08:20:55,199 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:55,204 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:55,205 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,208 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,208 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,208 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:20:55,210 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,213 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:55,214 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c52269ac3ef6cfb97100e50b1f62738b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70362014, jitterRate=0.0484757125377655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:20:55,215 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c52269ac3ef6cfb97100e50b1f62738b: Writing region info on filesystem at 1733732455187Initializing all the Stores at 1733732455190 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732455190Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732455190Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732455190Cleaning up temporary data from old regions at 1733732455208 (+18 ms)Region opened successfully at 1733732455215 (+7 ms) 2024-12-09T08:20:55,215 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c52269ac3ef6cfb97100e50b1f62738b, disabling compactions & flushes 2024-12-09T08:20:55,215 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:55,215 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:55,215 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. after waiting 0 ms 2024-12-09T08:20:55,215 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:55,221 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:55,221 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c52269ac3ef6cfb97100e50b1f62738b: Waiting for close lock at 1733732455215Disabling compacts and flushes for region at 1733732455215Disabling writes for close at 1733732455215Writing region close event to WAL at 1733732455221 (+6 ms)Closed at 1733732455221 2024-12-09T08:20:55,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741878_1055 (size=93) 2024-12-09T08:20:55,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741878_1055 (size=93) 2024-12-09T08:20:55,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741878_1055 (size=93) 2024-12-09T08:20:55,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:55,232 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70145694:(num 1733732455125) 2024-12-09T08:20:55,232 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:55,235 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:55,253 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235, exclude list is [], retry=0 2024-12-09T08:20:55,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:55,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:55,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:55,258 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 2024-12-09T08:20:55,259 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:20:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c52269ac3ef6cfb97100e50b1f62738b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,259 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,261 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,262 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName a 2024-12-09T08:20:55,262 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:55,262 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:55,262 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,263 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName b 2024-12-09T08:20:55,263 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:55,264 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:55,264 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,265 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName c 2024-12-09T08:20:55,265 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:55,265 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:55,265 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,266 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,267 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,268 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,268 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,269 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:20:55,270 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:55,270 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c52269ac3ef6cfb97100e50b1f62738b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61742607, jitterRate=-0.07996346056461334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:20:55,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c52269ac3ef6cfb97100e50b1f62738b: Writing region info on filesystem at 1733732455259Initializing all the Stores at 1733732455260 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732455260Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732455261 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732455261Cleaning up temporary data from old regions at 1733732455268 (+7 ms)Region opened successfully at 1733732455271 (+3 ms) 2024-12-09T08:20:55,275 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733732455274/Put/seqid=0 2024-12-09T08:20:55,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741881_1058 (size=4826) 2024-12-09T08:20:55,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741881_1058 (size=4826) 2024-12-09T08:20:55,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741881_1058 (size=4826) 2024-12-09T08:20:55,282 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40523/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in c52269ac3ef6cfb97100e50b1f62738b/a 2024-12-09T08:20:55,289 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T08:20:55,289 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:20:55,290 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c52269ac3ef6cfb97100e50b1f62738b: 2024-12-09T08:20:55,291 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_ 2024-12-09T08:20:55,292 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40523/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into c52269ac3ef6cfb97100e50b1f62738b/a as hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_ - updating store file list. 2024-12-09T08:20:55,298 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T08:20:55,299 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_ into c52269ac3ef6cfb97100e50b1f62738b/a 2024-12-09T08:20:55,299 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40523/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into c52269ac3ef6cfb97100e50b1f62738b/a (new location: hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_) 2024-12-09T08:20:55,343 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235, size=0 (0bytes) 2024-12-09T08:20:55,344 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 might be still open, length is 0 2024-12-09T08:20:55,344 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 2024-12-09T08:20:55,344 WARN [IPC Server handler 3 on default port 40523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T08:20:55,344 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 after 0ms 2024-12-09T08:20:57,864 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:56072 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38773:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56072 dst: /127.0.0.1:38773 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38773 remote=/127.0.0.1:56072]. Total timeout mills is 60000, 57441 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:57,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:34834 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34834 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:57,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:54074 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41795:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54074 dst: /127.0.0.1:41795 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:20:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741880_1059 (size=434) 2024-12-09T08:20:57,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741880_1059 (size=434) 2024-12-09T08:20:59,346 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 after 4001ms 2024-12-09T08:20:59,351 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:59,351 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 took 4008ms 2024-12-09T08:20:59,355 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235; continuing. 2024-12-09T08:20:59,355 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 so closing down 2024-12-09T08:20:59,355 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:20:59,358 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733732455235.temp 2024-12-09T08:20:59,359 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005-wal.1733732455235.temp 2024-12-09T08:20:59,360 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:20:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741882_1060 (size=236) 2024-12-09T08:20:59,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741882_1060 (size=236) 2024-12-09T08:20:59,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741882_1060 (size=236) 2024-12-09T08:20:59,368 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005-wal.1733732455235.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T08:20:59,370 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005-wal.1733732455235.temp to hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005 2024-12-09T08:20:59,370 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:20:59,370 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235, journal: Splitting hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235, size=0 (0bytes) at 1733732455344Finishing writing output for hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 so closing down at 1733732459355 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005-wal.1733732455235.temp at 1733732459359 (+4 ms)3 split writer threads finished at 1733732459360 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005-wal.1733732455235.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733732459368 (+8 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005-wal.1733732455235.temp to hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005 at 1733732459370 (+2 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235, size=0, length=0, corrupted=false, cancelled=false at 1733732459370 2024-12-09T08:20:59,372 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732455235 2024-12-09T08:20:59,373 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005 2024-12-09T08:20:59,373 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:20:59,375 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:20:59,391 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732459376, exclude list is [], retry=0 2024-12-09T08:20:59,393 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:20:59,394 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:20:59,394 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:20:59,395 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732459376 2024-12-09T08:20:59,396 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:20:59,396 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c52269ac3ef6cfb97100e50b1f62738b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:59,396 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:59,396 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,396 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,397 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,398 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName a 2024-12-09T08:20:59,398 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:59,404 DEBUG [StoreFileOpener-c52269ac3ef6cfb97100e50b1f62738b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T08:20:59,404 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/32f801eb37cf44b68c4bce422fe1a2d1_SeqId_3_ 2024-12-09T08:20:59,404 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:59,404 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,406 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName b 2024-12-09T08:20:59,406 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:59,406 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:59,406 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,407 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c52269ac3ef6cfb97100e50b1f62738b columnFamilyName c 2024-12-09T08:20:59,407 DEBUG [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:59,408 INFO [StoreOpener-c52269ac3ef6cfb97100e50b1f62738b-1 {}] regionserver.HStore(327): Store=c52269ac3ef6cfb97100e50b1f62738b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:59,408 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,411 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005 2024-12-09T08:20:59,413 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:20:59,414 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005 2024-12-09T08:20:59,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c52269ac3ef6cfb97100e50b1f62738b 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T08:20:59,427 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/.tmp/a/55f10432f8254f6f86f1dffab36417f2 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733732455303/Put/seqid=0 2024-12-09T08:20:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741884_1062 (size=5149) 2024-12-09T08:20:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741884_1062 (size=5149) 2024-12-09T08:20:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741884_1062 (size=5149) 2024-12-09T08:20:59,434 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/.tmp/a/55f10432f8254f6f86f1dffab36417f2 2024-12-09T08:20:59,439 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/.tmp/a/55f10432f8254f6f86f1dffab36417f2 as hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/55f10432f8254f6f86f1dffab36417f2 2024-12-09T08:20:59,445 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/a/55f10432f8254f6f86f1dffab36417f2, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T08:20:59,445 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for c52269ac3ef6cfb97100e50b1f62738b in 31ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T08:20:59,446 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/0000000000000000005 2024-12-09T08:20:59,447 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,447 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,448 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:20:59,449 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c52269ac3ef6cfb97100e50b1f62738b 2024-12-09T08:20:59,451 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/c52269ac3ef6cfb97100e50b1f62738b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T08:20:59,452 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c52269ac3ef6cfb97100e50b1f62738b; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63318498, jitterRate=-0.056480854749679565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:20:59,452 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c52269ac3ef6cfb97100e50b1f62738b: Writing region info on filesystem at 1733732459396Initializing all the Stores at 1733732459397 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732459397Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732459397Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732459397Obtaining lock to block concurrent updates at 1733732459414 (+17 ms)Preparing flush snapshotting stores in c52269ac3ef6cfb97100e50b1f62738b at 1733732459414Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733732459415 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. at 1733732459415Flushing c52269ac3ef6cfb97100e50b1f62738b/a: creating writer at 1733732459415Flushing c52269ac3ef6cfb97100e50b1f62738b/a: appending metadata at 1733732459427 (+12 ms)Flushing c52269ac3ef6cfb97100e50b1f62738b/a: closing flushed file at 1733732459427Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e7753f8: reopening flushed file at 1733732459438 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for c52269ac3ef6cfb97100e50b1f62738b in 31ms, sequenceid=5, compaction requested=false; wal=null at 1733732459445 (+7 ms)Cleaning up temporary data from old regions at 1733732459447 (+2 ms)Region opened successfully at 1733732459452 (+5 ms) 2024-12-09T08:20:59,455 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c52269ac3ef6cfb97100e50b1f62738b, disabling compactions & flushes 2024-12-09T08:20:59,455 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:59,455 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:59,455 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. after waiting 0 ms 2024-12-09T08:20:59,455 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:59,456 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733732455100.c52269ac3ef6cfb97100e50b1f62738b. 2024-12-09T08:20:59,456 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c52269ac3ef6cfb97100e50b1f62738b: Waiting for close lock at 1733732459455Disabling compacts and flushes for region at 1733732459455Disabling writes for close at 1733732459455Writing region close event to WAL at 1733732459456 (+1 ms)Closed at 1733732459456 2024-12-09T08:20:59,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741883_1061 (size=93) 2024-12-09T08:20:59,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741883_1061 (size=93) 2024-12-09T08:20:59,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741883_1061 (size=93) 2024-12-09T08:20:59,462 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:20:59,462 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732459376) 2024-12-09T08:20:59,473 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=419 (was 413) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_485223475_22 at /127.0.0.1:56082 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_485223475_22 at /127.0.0.1:34880 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_485223475_22 at /127.0.0.1:54112 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40523 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40523 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 252) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4086 (was 4169) 2024-12-09T08:20:59,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=419, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=4086 2024-12-09T08:20:59,496 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:20:59,501 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:20:59,504 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,40865,1733732443726 2024-12-09T08:20:59,506 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4366b3ef 2024-12-09T08:20:59,507 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:20:59,509 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:20:59,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:20:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T08:20:59,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:20:59,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T08:20:59,522 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:59,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:20:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:20:59,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741885_1063 (size=694) 2024-12-09T08:20:59,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741885_1063 (size=694) 2024-12-09T08:20:59,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741885_1063 (size=694) 2024-12-09T08:20:59,536 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609 2024-12-09T08:20:59,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741886_1064 (size=77) 2024-12-09T08:20:59,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741886_1064 (size=77) 2024-12-09T08:20:59,545 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:59,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741886_1064 (size=77) 2024-12-09T08:20:59,545 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:20:59,545 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,545 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,546 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:20:59,546 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,546 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,546 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732459545Disabling compacts and flushes for region at 1733732459545Disabling writes for close at 1733732459546 (+1 ms)Writing region close event to WAL at 1733732459546Closed at 1733732459546 2024-12-09T08:20:59,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:20:59,553 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733732459547"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732459547"}]},"ts":"1733732459547"} 2024-12-09T08:20:59,557 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:20:59,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:20:59,561 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732459559"}]},"ts":"1733732459559"} 2024-12-09T08:20:59,565 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T08:20:59,565 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {5bda946059d1=0} racks are {/default-rack=0} 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T08:20:59,567 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T08:20:59,567 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T08:20:59,567 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T08:20:59,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T08:20:59,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN}] 2024-12-09T08:20:59,570 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN 2024-12-09T08:20:59,571 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN; state=OFFLINE, location=5bda946059d1,41695,1733732444707; forceNewPlan=false, retain=false 2024-12-09T08:20:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:20:59,723 INFO [5bda946059d1:40865 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T08:20:59,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPENING, regionLocation=5bda946059d1,41695,1733732444707 2024-12-09T08:20:59,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN because future has completed 2024-12-09T08:20:59,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707}] 2024-12-09T08:20:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:20:59,882 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:20:59,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45669, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:20:59,894 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,894 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:20:59,895 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,895 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:20:59,896 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,896 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,898 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,901 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf1 2024-12-09T08:20:59,901 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:59,902 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:59,902 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,905 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf2 2024-12-09T08:20:59,905 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:20:59,905 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:20:59,906 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,907 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,908 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,909 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,909 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,911 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:20:59,913 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,917 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:20:59,917 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 920bbb1c4e65110d65293a979c5a15b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61122358, jitterRate=-0.08920589089393616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:20:59,918 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:20:59,918 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 920bbb1c4e65110d65293a979c5a15b1: Running coprocessor pre-open hook at 1733732459896Writing region info on filesystem at 1733732459896Initializing all the Stores at 1733732459898 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732459898Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732459898Cleaning up temporary data from old regions at 1733732459909 (+11 ms)Running coprocessor post-open hooks at 1733732459918 (+9 ms)Region opened successfully at 1733732459918 2024-12-09T08:20:59,920 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., pid=6, masterSystemTime=1733732459881 2024-12-09T08:20:59,923 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,923 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:20:59,924 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,41695,1733732444707 2024-12-09T08:20:59,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707 because future has completed 2024-12-09T08:20:59,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T08:20:59,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707 in 200 msec 2024-12-09T08:20:59,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T08:20:59,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN in 363 msec 2024-12-09T08:20:59,936 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:20:59,936 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732459936"}]},"ts":"1733732459936"} 2024-12-09T08:20:59,938 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T08:20:59,939 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:20:59,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 427 msec 2024-12-09T08:21:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:21:00,158 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T08:21:00,158 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T08:21:00,161 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T08:21:00,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T08:21:00,171 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T08:21:00,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T08:21:00,180 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=2] 2024-12-09T08:21:00,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:21:00,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:21:00,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=920bbb1c4e65110d65293a979c5a15b1, source=5bda946059d1,41695,1733732444707, destination=5bda946059d1,41239,1733732444531, warming up region on 5bda946059d1,41239,1733732444531 2024-12-09T08:21:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:21:00,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=920bbb1c4e65110d65293a979c5a15b1, source=5bda946059d1,41695,1733732444707, destination=5bda946059d1,41239,1733732444531, running balancer 2024-12-09T08:21:00,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:21:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE 2024-12-09T08:21:00,202 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE 2024-12-09T08:21:00,204 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=CLOSING, regionLocation=5bda946059d1,41695,1733732444707 2024-12-09T08:21:00,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(7855): Warmup {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:00,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE because future has completed 2024-12-09T08:21:00,207 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:21:00,207 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707}] 2024-12-09T08:21:00,209 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf1 2024-12-09T08:21:00,209 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:00,210 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:00,210 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,211 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf2 2024-12-09T08:21:00,211 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:00,212 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:21:00,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:21:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732460212Disabling compacts and flushes for region at 1733732460212Disabling writes for close at 1733732460212Writing region close event to WAL at 1733732460212Closed at 1733732460213 (+1 ms) 2024-12-09T08:21:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T08:21:00,365 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,365 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:21:00,366 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:21:00,366 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,366 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,366 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:21:00,366 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,366 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 920bbb1c4e65110d65293a979c5a15b1 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T08:21:00,382 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/23639b133520471a9dee33a4b34a239a is 35, key is r1/cf1:q/1733732460185/Put/seqid=0 2024-12-09T08:21:00,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741887_1065 (size=4783) 2024-12-09T08:21:00,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741887_1065 (size=4783) 2024-12-09T08:21:00,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741887_1065 (size=4783) 2024-12-09T08:21:00,392 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/23639b133520471a9dee33a4b34a239a 2024-12-09T08:21:00,399 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/23639b133520471a9dee33a4b34a239a as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a 2024-12-09T08:21:00,405 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T08:21:00,406 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 920bbb1c4e65110d65293a979c5a15b1 in 40ms, sequenceid=5, compaction requested=false 2024-12-09T08:21:00,407 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T08:21:00,411 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T08:21:00,414 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,414 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732460366Running coprocessor pre-close hooks at 1733732460366Disabling compacts and flushes for region at 1733732460366Disabling writes for close at 1733732460366Obtaining lock to block concurrent updates at 1733732460366Preparing flush snapshotting stores in 920bbb1c4e65110d65293a979c5a15b1 at 1733732460366Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733732460367 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. at 1733732460367Flushing 920bbb1c4e65110d65293a979c5a15b1/cf1: creating writer at 1733732460368 (+1 ms)Flushing 920bbb1c4e65110d65293a979c5a15b1/cf1: appending metadata at 1733732460381 (+13 ms)Flushing 920bbb1c4e65110d65293a979c5a15b1/cf1: closing flushed file at 1733732460381Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f65c4fe: reopening flushed file at 1733732460398 (+17 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 920bbb1c4e65110d65293a979c5a15b1 in 40ms, sequenceid=5, compaction requested=false at 1733732460406 (+8 ms)Writing region close event to WAL at 1733732460408 (+2 ms)Running coprocessor post-close hooks at 1733732460412 (+4 ms)Closed at 1733732460414 (+2 ms) 2024-12-09T08:21:00,415 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 920bbb1c4e65110d65293a979c5a15b1 move to 5bda946059d1,41239,1733732444531 record at close sequenceid=5 2024-12-09T08:21:00,418 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,419 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=CLOSED 2024-12-09T08:21:00,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707 because future has completed 2024-12-09T08:21:00,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T08:21:00,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707 in 216 msec 2024-12-09T08:21:00,427 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,41239,1733732444531; forceNewPlan=false, retain=false 2024-12-09T08:21:00,577 INFO [5bda946059d1:40865 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T08:21:00,577 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPENING, regionLocation=5bda946059d1,41239,1733732444531 2024-12-09T08:21:00,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE because future has completed 2024-12-09T08:21:00,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531}] 2024-12-09T08:21:00,745 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,746 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:00,747 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,747 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:00,747 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,747 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,750 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,751 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf1 2024-12-09T08:21:00,751 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:00,760 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a 2024-12-09T08:21:00,760 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:00,760 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,761 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf2 2024-12-09T08:21:00,761 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:00,762 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:00,762 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,763 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,764 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,765 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,765 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,765 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:21:00,767 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,768 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 920bbb1c4e65110d65293a979c5a15b1; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70667817, jitterRate=0.0530325323343277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:21:00,768 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:00,769 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 920bbb1c4e65110d65293a979c5a15b1: Running coprocessor pre-open hook at 1733732460748Writing region info on filesystem at 1733732460748Initializing all the Stores at 1733732460749 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732460749Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732460749Cleaning up temporary data from old regions at 1733732460765 (+16 ms)Running coprocessor post-open hooks at 1733732460768 (+3 ms)Region opened successfully at 1733732460769 (+1 ms) 2024-12-09T08:21:00,770 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., pid=9, masterSystemTime=1733732460734 2024-12-09T08:21:00,772 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,772 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:00,773 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPEN, openSeqNum=9, regionLocation=5bda946059d1,41239,1733732444531 2024-12-09T08:21:00,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531 because future has completed 2024-12-09T08:21:00,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T08:21:00,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531 in 197 msec 2024-12-09T08:21:00,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE in 581 msec 2024-12-09T08:21:00,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:21:00,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:21:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:42030 deadline: 1733732520814, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41239 startCode=1733732444531. As of locationSeqNum=5. 2024-12-09T08:21:00,835 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41239 startCode=1733732444531. As of locationSeqNum=5. 2024-12-09T08:21:00,836 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41239 startCode=1733732444531. As of locationSeqNum=5. 2024-12-09T08:21:00,836 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41239,1733732444531, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41239 startCode=1733732444531. As of locationSeqNum=5. 2024-12-09T08:21:00,945 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:21:00,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:21:00,957 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 920bbb1c4e65110d65293a979c5a15b1 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T08:21:00,973 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a is 29, key is r1/cf1:/1733732460949/DeleteFamily/seqid=0 2024-12-09T08:21:00,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741888_1066 (size=4906) 2024-12-09T08:21:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741888_1066 (size=4906) 2024-12-09T08:21:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741888_1066 (size=4906) 2024-12-09T08:21:00,980 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:00,986 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:00,998 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf2/bfff049734644420a31f04bef021961d is 29, key is r1/cf2:/1733732460949/DeleteFamily/seqid=0 2024-12-09T08:21:01,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741889_1067 (size=4906) 2024-12-09T08:21:01,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741889_1067 (size=4906) 2024-12-09T08:21:01,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741889_1067 (size=4906) 2024-12-09T08:21:01,411 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf2/bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,423 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,425 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:01,433 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:01,433 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T08:21:01,434 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf2/bfff049734644420a31f04bef021961d as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,441 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,441 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T08:21:01,442 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 920bbb1c4e65110d65293a979c5a15b1 in 485ms, sequenceid=12, compaction requested=false 2024-12-09T08:21:01,442 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 920bbb1c4e65110d65293a979c5a15b1: 2024-12-09T08:21:01,445 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T08:21:01,447 DEBUG [Time-limited test {}] regionserver.HStore(1541): 920bbb1c4e65110d65293a979c5a15b1/cf1 is initiating major compaction (all files) 2024-12-09T08:21:01,447 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:21:01,447 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:21:01,447 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 920bbb1c4e65110d65293a979c5a15b1/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,448 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a, hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a] into tmpdir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp, totalSize=9.5 K 2024-12-09T08:21:01,449 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 23639b133520471a9dee33a4b34a239a, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733732460185 2024-12-09T08:21:01,449 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2b41ddd3d1324d31ae279e1e0cf9db0a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T08:21:01,460 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 920bbb1c4e65110d65293a979c5a15b1#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:21:01,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741890_1068 (size=4626) 2024-12-09T08:21:01,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741890_1068 (size=4626) 2024-12-09T08:21:01,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741890_1068 (size=4626) 2024-12-09T08:21:01,473 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf1/1322f01e32484a4ea6b39b62535a0d50 as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/1322f01e32484a4ea6b39b62535a0d50 2024-12-09T08:21:01,487 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 920bbb1c4e65110d65293a979c5a15b1/cf1 of 920bbb1c4e65110d65293a979c5a15b1 into 1322f01e32484a4ea6b39b62535a0d50(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:21:01,487 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 920bbb1c4e65110d65293a979c5a15b1: 2024-12-09T08:21:01,487 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T08:21:01,487 DEBUG [Time-limited test {}] regionserver.HStore(1541): 920bbb1c4e65110d65293a979c5a15b1/cf2 is initiating major compaction (all files) 2024-12-09T08:21:01,487 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:21:01,487 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:21:01,487 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 920bbb1c4e65110d65293a979c5a15b1/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,487 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d] into tmpdir=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp, totalSize=4.8 K 2024-12-09T08:21:01,488 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bfff049734644420a31f04bef021961d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T08:21:01,493 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 920bbb1c4e65110d65293a979c5a15b1#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:21:01,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741891_1069 (size=4592) 2024-12-09T08:21:01,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741891_1069 (size=4592) 2024-12-09T08:21:01,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741891_1069 (size=4592) 2024-12-09T08:21:01,512 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/.tmp/cf2/ff0a43abc6da445088449179c946aae4 as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/ff0a43abc6da445088449179c946aae4 2024-12-09T08:21:01,519 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 920bbb1c4e65110d65293a979c5a15b1/cf2 of 920bbb1c4e65110d65293a979c5a15b1 into ff0a43abc6da445088449179c946aae4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:21:01,519 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 920bbb1c4e65110d65293a979c5a15b1: 2024-12-09T08:21:01,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=920bbb1c4e65110d65293a979c5a15b1, source=5bda946059d1,41239,1733732444531, destination=5bda946059d1,41695,1733732444707, warming up region on 5bda946059d1,41695,1733732444707 2024-12-09T08:21:01,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=920bbb1c4e65110d65293a979c5a15b1, source=5bda946059d1,41239,1733732444531, destination=5bda946059d1,41695,1733732444707, running balancer 2024-12-09T08:21:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE 2024-12-09T08:21:01,525 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE 2024-12-09T08:21:01,526 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=CLOSING, regionLocation=5bda946059d1,41239,1733732444531 2024-12-09T08:21:01,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(7855): Warmup {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:01,528 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:01,529 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf1 2024-12-09T08:21:01,529 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:01,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE because future has completed 2024-12-09T08:21:01,530 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:21:01,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531}] 2024-12-09T08:21:01,537 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/1322f01e32484a4ea6b39b62535a0d50 2024-12-09T08:21:01,543 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a 2024-12-09T08:21:01,549 INFO [StoreFileOpener-920bbb1c4e65110d65293a979c5a15b1-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:01,549 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:01,549 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:01,549 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:01,551 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf2 2024-12-09T08:21:01,551 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:01,557 INFO [StoreFileOpener-920bbb1c4e65110d65293a979c5a15b1-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,557 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,562 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/ff0a43abc6da445088449179c946aae4 2024-12-09T08:21:01,562 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:21:01,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:21:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41695 {}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732461563Disabling compacts and flushes for region at 1733732461563Disabling writes for close at 1733732461563Writing region close event to WAL at 1733732461564 (+1 ms)Closed at 1733732461564 2024-12-09T08:21:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T08:21:01,684 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:01,684 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:21:01,684 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:21:01,685 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,685 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,685 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:21:01,685 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,685 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a, hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a] to archive 2024-12-09T08:21:01,688 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:21:01,692 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a to hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/23639b133520471a9dee33a4b34a239a 2024-12-09T08:21:01,693 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a to hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/2b41ddd3d1324d31ae279e1e0cf9db0a 2024-12-09T08:21:01,704 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d] to archive 2024-12-09T08:21:01,705 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:21:01,708 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d to hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/bfff049734644420a31f04bef021961d 2024-12-09T08:21:01,713 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T08:21:01,714 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:01,714 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732461684Running coprocessor pre-close hooks at 1733732461684Disabling compacts and flushes for region at 1733732461684Disabling writes for close at 1733732461685 (+1 ms)Writing region close event to WAL at 1733732461709 (+24 ms)Running coprocessor post-close hooks at 1733732461713 (+4 ms)Closed at 1733732461713 2024-12-09T08:21:01,714 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 920bbb1c4e65110d65293a979c5a15b1 move to 5bda946059d1,41695,1733732444707 record at close sequenceid=12 2024-12-09T08:21:01,716 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:01,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=CLOSED 2024-12-09T08:21:01,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531 because future has completed 2024-12-09T08:21:01,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T08:21:01,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531 in 191 msec 2024-12-09T08:21:01,725 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,41695,1733732444707; forceNewPlan=false, retain=false 2024-12-09T08:21:01,876 INFO [5bda946059d1:40865 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T08:21:01,877 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPENING, regionLocation=5bda946059d1,41695,1733732444707 2024-12-09T08:21:01,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE because future has completed 2024-12-09T08:21:01,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707}] 2024-12-09T08:21:02,043 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,043 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:02,044 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,044 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:02,044 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,044 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,046 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,048 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf1 2024-12-09T08:21:02,048 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:02,054 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/1322f01e32484a4ea6b39b62535a0d50 2024-12-09T08:21:02,054 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:02,054 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,055 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf2 2024-12-09T08:21:02,055 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:02,061 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/ff0a43abc6da445088449179c946aae4 2024-12-09T08:21:02,061 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:02,061 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,062 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,063 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,063 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,064 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,064 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:21:02,066 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,067 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 920bbb1c4e65110d65293a979c5a15b1; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59212994, jitterRate=-0.1176576316356659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:21:02,067 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,068 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 920bbb1c4e65110d65293a979c5a15b1: Running coprocessor pre-open hook at 1733732462044Writing region info on filesystem at 1733732462044Initializing all the Stores at 1733732462046 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732462046Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732462046Cleaning up temporary data from old regions at 1733732462064 (+18 ms)Running coprocessor post-open hooks at 1733732462067 (+3 ms)Region opened successfully at 1733732462068 (+1 ms) 2024-12-09T08:21:02,069 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., pid=12, masterSystemTime=1733732462038 2024-12-09T08:21:02,072 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,072 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,073 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPEN, openSeqNum=18, regionLocation=5bda946059d1,41695,1733732444707 2024-12-09T08:21:02,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707 because future has completed 2024-12-09T08:21:02,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T08:21:02,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41695,1733732444707 in 196 msec 2024-12-09T08:21:02,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, REOPEN/MOVE in 557 msec 2024-12-09T08:21:02,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:21:02,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:21:02,133 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 5bda946059d1,41695,1733732444707: testing ***** 2024-12-09T08:21:02,134 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T08:21:02,137 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T08:21:02,138 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T08:21:02,140 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T08:21:02,141 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T08:21:02,148 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 319585944 }, "NonHeapMemoryUsage": { "committed": 171114496, "init": 7667712, "max": -1, "used": 168536464 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "5bda946059d1", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "5bda946059d1", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61985", "tag.serverName": "5bda946059d1,41239,1733732444531", "tag.clusterId": "b0508df1-95ca-4311-b15c-3eff9ce222f2", "tag.Context": "regionserver", "tag.Hostname": "5bda946059d1", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733732444531, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T08:21:02,151 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40865 {}] master.MasterRpcServices(700): 5bda946059d1,41695,1733732444707 reported a fatal error: ***** ABORTING region server 5bda946059d1,41695,1733732444707: testing ***** 2024-12-09T08:21:02,153 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,41695,1733732444707' ***** 2024-12-09T08:21:02,153 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T08:21:02,154 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:21:02,154 INFO [RS:2;5bda946059d1:41695 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T08:21:02,154 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:21:02,154 INFO [RS:2;5bda946059d1:41695 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T08:21:02,154 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(3091): Received CLOSE for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:56738 deadline: 1733732522154, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41695 startCode=1733732444707. As of locationSeqNum=12. 2024-12-09T08:21:02,155 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(956): aborting server 5bda946059d1,41695,1733732444707 2024-12-09T08:21:02,155 INFO [RS:2;5bda946059d1:41695 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:21:02,155 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:21:02,155 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41239,1733732444531, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41239,1733732444531, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41695 startCode=1733732444707. As of locationSeqNum=12. 2024-12-09T08:21:02,155 INFO [RS:2;5bda946059d1:41695 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;5bda946059d1:41695. 2024-12-09T08:21:02,155 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,155 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41239,1733732444531, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41695 startCode=1733732444707. As of locationSeqNum=12. 2024-12-09T08:21:02,155 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,155 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41239,1733732444531, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=41695 startCode=1733732444707. As of locationSeqNum=12. 2024-12-09T08:21:02,155 DEBUG [RS:2;5bda946059d1:41695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:21:02,155 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:21:02,155 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,156 DEBUG [RS:2;5bda946059d1:41695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:02,156 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:21:02,156 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1325): Online Regions={920bbb1c4e65110d65293a979c5a15b1=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.} 2024-12-09T08:21:02,157 DEBUG [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1351): Waiting on 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:02,157 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,157 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732462155Running coprocessor pre-close hooks at 1733732462155Disabling compacts and flushes for region at 1733732462155Disabling writes for close at 1733732462155Writing region close event to WAL at 1733732462157 (+2 ms)Running coprocessor post-close hooks at 1733732462157Closed at 1733732462157 2024-12-09T08:21:02,158 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:02,182 INFO [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T08:21:02,182 INFO [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T08:21:02,197 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:21:02,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5bda946059d1,41695,1733732444707 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:02,269 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5bda946059d1,41695,1733732444707 aborting 2024-12-09T08:21:02,269 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5bda946059d1,41695,1733732444707 aborting 2024-12-09T08:21:02,269 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=12 from cache 2024-12-09T08:21:02,357 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,41695,1733732444707; all regions closed. 2024-12-09T08:21:02,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741834_1010 (size=1677) 2024-12-09T08:21:02,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741834_1010 (size=1677) 2024-12-09T08:21:02,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741834_1010 (size=1677) 2024-12-09T08:21:02,365 DEBUG [RS:2;5bda946059d1:41695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:02,365 INFO [RS:2;5bda946059d1:41695 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:21:02,365 INFO [RS:2;5bda946059d1:41695 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:21:02,365 INFO [RS:2;5bda946059d1:41695 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T08:21:02,366 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:21:02,366 INFO [RS:2;5bda946059d1:41695 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:21:02,366 INFO [RS:2;5bda946059d1:41695 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:21:02,366 INFO [RS:2;5bda946059d1:41695 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:21:02,366 INFO [RS:2;5bda946059d1:41695 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:21:02,366 INFO [RS:2;5bda946059d1:41695 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41695 2024-12-09T08:21:02,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:21:02,439 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,41695,1733732444707 2024-12-09T08:21:02,439 INFO [RS:2;5bda946059d1:41695 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:21:02,442 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,41695,1733732444707] 2024-12-09T08:21:02,459 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,41695,1733732444707 already deleted, retry=false 2024-12-09T08:21:02,460 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 5bda946059d1,41695,1733732444707 on 5bda946059d1,40865,1733732443726 2024-12-09T08:21:02,467 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 5bda946059d1,41695,1733732444707, splitWal=true, meta=false 2024-12-09T08:21:02,470 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 5bda946059d1,41695,1733732444707, splitWal=true, meta=false 2024-12-09T08:21:02,471 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 5bda946059d1,41695,1733732444707 (carryingMeta=false) 5bda946059d1,41695,1733732444707/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2d99e799[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T08:21:02,473 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 5bda946059d1,41695,1733732444707 had 1 regions 2024-12-09T08:21:02,474 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 5bda946059d1,41695,1733732444707, splitWal=true, meta=false, isMeta: false 2024-12-09T08:21:02,476 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting 2024-12-09T08:21:02,477 INFO [PEWorker-2 {}] master.SplitWALManager(105): 5bda946059d1,41695,1733732444707 WAL count=1, meta=false 2024-12-09T08:21:02,479 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18] 2024-12-09T08:21:02,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5bda946059d1%2C41695%2C1733732444707.1733732446346}] 2024-12-09T08:21:02,480 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5bda946059d1:41695 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5bda946059d1/172.17.0.2:41695 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:02,481 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18, error=java.net.ConnectException: Call to address=5bda946059d1:41695 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5bda946059d1/172.17.0.2:41695 2024-12-09T08:21:02,481 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T08:21:02,481 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18 from cache 2024-12-09T08:21:02,482 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 5bda946059d1:41695 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5bda946059d1/172.17.0.2:41695 2024-12-09T08:21:02,484 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=5bda946059d1,41239,1733732444531 2024-12-09T08:21:02,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5bda946059d1%2C41695%2C1733732444707.1733732446346, worker=5bda946059d1,41239,1733732444531}] 2024-12-09T08:21:02,550 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:02,550 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41695-0x10009a9aa0d0003, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:02,550 INFO [RS:2;5bda946059d1:41695 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:21:02,551 INFO [RS:2;5bda946059d1:41695 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,41695,1733732444707; zookeeper connection closed. 2024-12-09T08:21:02,551 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28cfa4a2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28cfa4a2 2024-12-09T08:21:02,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41239 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T08:21:02,660 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346, size=1.6 K (1677bytes) 2024-12-09T08:21:02,660 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 2024-12-09T08:21:02,661 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 after 1ms 2024-12-09T08:21:02,664 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:02,664 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 took 4ms 2024-12-09T08:21:02,669 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 920bbb1c4e65110d65293a979c5a15b1: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T08:21:02,670 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 so closing down 2024-12-09T08:21:02,670 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:02,670 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:02,670 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-09T08:21:02,670 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346, journal: Splitting hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346, size=1.6 K (1677bytes) at 1733732462660Finishing writing output for hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 so closing down at 1733732462670 (+10 ms)3 split writer threads finished at 1733732462670Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733732462670 2024-12-09T08:21:02,670 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 2024-12-09T08:21:02,671 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T08:21:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40865 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T08:21:02,677 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting/5bda946059d1%2C41695%2C1733732444707.1733732446346 to hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs 2024-12-09T08:21:02,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T08:21:02,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5bda946059d1%2C41695%2C1733732444707.1733732446346, worker=5bda946059d1,41239,1733732444531 in 192 msec 2024-12-09T08:21:02,682 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=5bda946059d1,41239,1733732444531 2024-12-09T08:21:02,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T08:21:02,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 5bda946059d1%2C41695%2C1733732444707.1733732446346, worker=5bda946059d1,41239,1733732444531 in 203 msec 2024-12-09T08:21:02,687 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting dir is empty, no logs to split. 2024-12-09T08:21:02,687 INFO [PEWorker-2 {}] master.SplitWALManager(105): 5bda946059d1,41695,1733732444707 WAL count=0, meta=false 2024-12-09T08:21:02,687 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 5bda946059d1,41695,1733732444707 WAL splitting is done? wals=0, meta=false 2024-12-09T08:21:02,690 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 5bda946059d1,41695,1733732444707 failed, ignore...File hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/WALs/5bda946059d1,41695,1733732444707-splitting does not exist. 2024-12-09T08:21:02,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN}] 2024-12-09T08:21:02,694 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN 2024-12-09T08:21:02,695 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T08:21:02,790 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18] 2024-12-09T08:21:02,791 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 5bda946059d1:41695 this server is in the failed servers list 2024-12-09T08:21:02,793 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=5bda946059d1:41695 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5bda946059d1:41695 2024-12-09T08:21:02,793 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5bda946059d1:41695 2024-12-09T08:21:02,793 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41695,1733732444707, seqNum=18 from cache 2024-12-09T08:21:02,846 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(204): Hosts are {5bda946059d1=0} racks are {/default-rack=0} 2024-12-09T08:21:02,846 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T08:21:02,846 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T08:21:02,846 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T08:21:02,846 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T08:21:02,846 INFO [5bda946059d1:40865 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T08:21:02,846 INFO [5bda946059d1:40865 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T08:21:02,846 DEBUG [5bda946059d1:40865 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T08:21:02,846 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPENING, regionLocation=5bda946059d1,41239,1733732444531 2024-12-09T08:21:02,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN because future has completed 2024-12-09T08:21:02,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531}] 2024-12-09T08:21:03,037 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:03,037 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 920bbb1c4e65110d65293a979c5a15b1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:03,037 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,038 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:03,038 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,038 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,040 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,041 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf1 2024-12-09T08:21:03,041 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,047 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf1/1322f01e32484a4ea6b39b62535a0d50 2024-12-09T08:21:03,048 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,048 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,049 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 920bbb1c4e65110d65293a979c5a15b1 columnFamilyName cf2 2024-12-09T08:21:03,049 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,055 DEBUG [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/cf2/ff0a43abc6da445088449179c946aae4 2024-12-09T08:21:03,055 INFO [StoreOpener-920bbb1c4e65110d65293a979c5a15b1-1 {}] regionserver.HStore(327): Store=920bbb1c4e65110d65293a979c5a15b1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,056 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,057 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,058 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,059 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,059 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,059 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:21:03,061 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,062 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 920bbb1c4e65110d65293a979c5a15b1; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75412565, jitterRate=0.12373478710651398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:21:03,062 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:03,062 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 920bbb1c4e65110d65293a979c5a15b1: Running coprocessor pre-open hook at 1733732463038Writing region info on filesystem at 1733732463038Initializing all the Stores at 1733732463039 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463039Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463039Cleaning up temporary data from old regions at 1733732463059 (+20 ms)Running coprocessor post-open hooks at 1733732463062 (+3 ms)Region opened successfully at 1733732463062 2024-12-09T08:21:03,063 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., pid=17, masterSystemTime=1733732463027 2024-12-09T08:21:03,065 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:03,065 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:03,066 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=920bbb1c4e65110d65293a979c5a15b1, regionState=OPEN, openSeqNum=18, regionLocation=5bda946059d1,41239,1733732444531 2024-12-09T08:21:03,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531 because future has completed 2024-12-09T08:21:03,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T08:21:03,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 920bbb1c4e65110d65293a979c5a15b1, server=5bda946059d1,41239,1733732444531 in 220 msec 2024-12-09T08:21:03,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T08:21:03,074 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 5bda946059d1,41695,1733732444707 after splitting done 2024-12-09T08:21:03,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=920bbb1c4e65110d65293a979c5a15b1, ASSIGN in 378 msec 2024-12-09T08:21:03,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 5bda946059d1,41695,1733732444707, splitWal=true, meta=false in 613 msec 2024-12-09T08:21:03,312 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1., hostname=5bda946059d1,41239,1733732444531, seqNum=18] 2024-12-09T08:21:03,331 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=413 (was 419), OpenFileDescriptor=1049 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 264), ProcessCount=11 (was 11), AvailableMemoryMB=4054 (was 4086) 2024-12-09T08:21:03,333 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-12-09T08:21:03,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=413, OpenFileDescriptor=1049, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=4053 2024-12-09T08:21:03,342 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-12-09T08:21:03,355 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:03,356 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:03,357 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:21:03,359 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63414475, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-63414475, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:03,370 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63414475/hregion-63414475.1733732463359, exclude list is [], retry=0 2024-12-09T08:21:03,372 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:03,372 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:03,372 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:03,376 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63414475/hregion-63414475.1733732463359 2024-12-09T08:21:03,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:21:03,377 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8d7b82a71238df1b2c2d653f64b9b46b, NAME => 'testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:21:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741893_1071 (size=67) 2024-12-09T08:21:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741893_1071 (size=67) 2024-12-09T08:21:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741893_1071 (size=67) 2024-12-09T08:21:03,385 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:03,386 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,387 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName a 2024-12-09T08:21:03,387 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,387 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,387 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,389 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName b 2024-12-09T08:21:03,389 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,389 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,389 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,391 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName c 2024-12-09T08:21:03,391 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,391 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,391 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,392 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,394 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:03,396 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,398 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:21:03,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d7b82a71238df1b2c2d653f64b9b46b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75351611, jitterRate=0.12282650172710419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:03,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d7b82a71238df1b2c2d653f64b9b46b: Writing region info on filesystem at 1733732463385Initializing all the Stores at 1733732463385Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463385Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463385Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463385Cleaning up temporary data from old regions at 1733732463394 (+9 ms)Region opened successfully at 1733732463399 (+5 ms) 2024-12-09T08:21:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d7b82a71238df1b2c2d653f64b9b46b, disabling compactions & flushes 2024-12-09T08:21:03,400 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. after waiting 0 ms 2024-12-09T08:21:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,400 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d7b82a71238df1b2c2d653f64b9b46b: Waiting for close lock at 1733732463399Disabling compacts and flushes for region at 1733732463399Disabling writes for close at 1733732463400 (+1 ms)Writing region close event to WAL at 1733732463400Closed at 1733732463400 2024-12-09T08:21:03,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741892_1070 (size=93) 2024-12-09T08:21:03,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741892_1070 (size=93) 2024-12-09T08:21:03,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741892_1070 (size=93) 2024-12-09T08:21:03,405 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:03,405 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63414475:(num 1733732463359) 2024-12-09T08:21:03,405 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:03,407 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:03,418 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407, exclude list is [], retry=0 2024-12-09T08:21:03,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:03,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:03,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:03,422 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 2024-12-09T08:21:03,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:21:03,423 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d7b82a71238df1b2c2d653f64b9b46b, NAME => 'testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:03,423 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:03,423 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,423 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,424 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,425 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName a 2024-12-09T08:21:03,425 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,426 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,426 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,427 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName b 2024-12-09T08:21:03,427 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,428 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,428 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,429 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName c 2024-12-09T08:21:03,429 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,429 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,429 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,430 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,431 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,432 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,432 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,433 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:03,434 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,435 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d7b82a71238df1b2c2d653f64b9b46b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70851784, jitterRate=0.05577385425567627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:03,435 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d7b82a71238df1b2c2d653f64b9b46b: Writing region info on filesystem at 1733732463423Initializing all the Stores at 1733732463424 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463424Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463424Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463424Cleaning up temporary data from old regions at 1733732463432 (+8 ms)Region opened successfully at 1733732463435 (+3 ms) 2024-12-09T08:21:03,457 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d7b82a71238df1b2c2d653f64b9b46b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T08:21:03,472 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/a/9be3201ab3f54516bd4325be51a55723 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733732463435/Put/seqid=0 2024-12-09T08:21:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741895_1073 (size=5958) 2024-12-09T08:21:03,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741895_1073 (size=5958) 2024-12-09T08:21:03,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741895_1073 (size=5958) 2024-12-09T08:21:03,478 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/a/9be3201ab3f54516bd4325be51a55723 2024-12-09T08:21:03,497 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/b/e4ef2ff96a5842eea82847e2bc62128e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733732463443/Put/seqid=0 2024-12-09T08:21:03,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741896_1074 (size=5958) 2024-12-09T08:21:03,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741896_1074 (size=5958) 2024-12-09T08:21:03,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/b/e4ef2ff96a5842eea82847e2bc62128e 2024-12-09T08:21:03,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741896_1074 (size=5958) 2024-12-09T08:21:03,520 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/c/031db7e2820b4f02878528005086dac7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733732463449/Put/seqid=0 2024-12-09T08:21:03,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741897_1075 (size=5958) 2024-12-09T08:21:03,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741897_1075 (size=5958) 2024-12-09T08:21:03,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741897_1075 (size=5958) 2024-12-09T08:21:03,528 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/c/031db7e2820b4f02878528005086dac7 2024-12-09T08:21:03,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/a/9be3201ab3f54516bd4325be51a55723 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/a/9be3201ab3f54516bd4325be51a55723 2024-12-09T08:21:03,540 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/a/9be3201ab3f54516bd4325be51a55723, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T08:21:03,541 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/b/e4ef2ff96a5842eea82847e2bc62128e as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/b/e4ef2ff96a5842eea82847e2bc62128e 2024-12-09T08:21:03,547 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/b/e4ef2ff96a5842eea82847e2bc62128e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T08:21:03,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/c/031db7e2820b4f02878528005086dac7 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/c/031db7e2820b4f02878528005086dac7 2024-12-09T08:21:03,554 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/c/031db7e2820b4f02878528005086dac7, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T08:21:03,555 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8d7b82a71238df1b2c2d653f64b9b46b in 98ms, sequenceid=33, compaction requested=false 2024-12-09T08:21:03,555 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d7b82a71238df1b2c2d653f64b9b46b: 2024-12-09T08:21:03,555 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d7b82a71238df1b2c2d653f64b9b46b, disabling compactions & flushes 2024-12-09T08:21:03,555 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,555 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,555 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. after waiting 0 ms 2024-12-09T08:21:03,555 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,557 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. 2024-12-09T08:21:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d7b82a71238df1b2c2d653f64b9b46b: Waiting for close lock at 1733732463555Disabling compacts and flushes for region at 1733732463555Disabling writes for close at 1733732463555Writing region close event to WAL at 1733732463556 (+1 ms)Closed at 1733732463556 2024-12-09T08:21:03,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741894_1072 (size=2873) 2024-12-09T08:21:03,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741894_1072 (size=2873) 2024-12-09T08:21:03,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741894_1072 (size=2873) 2024-12-09T08:21:03,564 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/b/e4ef2ff96a5842eea82847e2bc62128e to hdfs://localhost:40523/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/b/e4ef2ff96a5842eea82847e2bc62128e 2024-12-09T08:21:03,578 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407, size=2.8 K (2873bytes) 2024-12-09T08:21:03,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 2024-12-09T08:21:03,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 after 1ms 2024-12-09T08:21:03,581 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:03,581 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 took 3ms 2024-12-09T08:21:03,583 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 so closing down 2024-12-09T08:21:03,583 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:03,584 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732463407.temp 2024-12-09T08:21:03,585 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000003-wal.1733732463407.temp 2024-12-09T08:21:03,585 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:03,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741898_1076 (size=2312) 2024-12-09T08:21:03,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741898_1076 (size=2312) 2024-12-09T08:21:03,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741898_1076 (size=2312) 2024-12-09T08:21:03,592 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000003-wal.1733732463407.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T08:21:03,593 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000003-wal.1733732463407.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032 2024-12-09T08:21:03,593 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-09T08:21:03,593 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407, journal: Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407, size=2.8 K (2873bytes) at 1733732463578Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 so closing down at 1733732463583 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000003-wal.1733732463407.temp at 1733732463585 (+2 ms)3 split writer threads finished at 1733732463585Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000003-wal.1733732463407.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733732463592 (+7 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000003-wal.1733732463407.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032 at 1733732463593 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733732463593 2024-12-09T08:21:03,595 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463407 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732463407 2024-12-09T08:21:03,595 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032 2024-12-09T08:21:03,595 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:03,597 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:03,608 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463597, exclude list is [], retry=0 2024-12-09T08:21:03,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:03,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:03,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:03,613 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463597 2024-12-09T08:21:03,613 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:21:03,613 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d7b82a71238df1b2c2d653f64b9b46b, NAME => 'testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:03,614 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:03,614 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,614 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,615 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,617 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName a 2024-12-09T08:21:03,617 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,622 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/a/9be3201ab3f54516bd4325be51a55723 2024-12-09T08:21:03,622 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,623 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,624 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName b 2024-12-09T08:21:03,624 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,625 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,625 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,626 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d7b82a71238df1b2c2d653f64b9b46b columnFamilyName c 2024-12-09T08:21:03,626 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,633 DEBUG [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/c/031db7e2820b4f02878528005086dac7 2024-12-09T08:21:03,633 INFO [StoreOpener-8d7b82a71238df1b2c2d653f64b9b46b-1 {}] regionserver.HStore(327): Store=8d7b82a71238df1b2c2d653f64b9b46b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,633 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,634 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,635 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,636 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032 2024-12-09T08:21:03,638 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:03,640 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032 2024-12-09T08:21:03,641 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d7b82a71238df1b2c2d653f64b9b46b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T08:21:03,654 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/b/e530e52bf67a486da83f41bcbe8b4f95 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733732463443/Put/seqid=0 2024-12-09T08:21:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741900_1078 (size=5958) 2024-12-09T08:21:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741900_1078 (size=5958) 2024-12-09T08:21:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741900_1078 (size=5958) 2024-12-09T08:21:03,660 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/b/e530e52bf67a486da83f41bcbe8b4f95 2024-12-09T08:21:03,667 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/.tmp/b/e530e52bf67a486da83f41bcbe8b4f95 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/b/e530e52bf67a486da83f41bcbe8b4f95 2024-12-09T08:21:03,672 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/b/e530e52bf67a486da83f41bcbe8b4f95, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T08:21:03,672 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8d7b82a71238df1b2c2d653f64b9b46b in 32ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T08:21:03,673 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/0000000000000000032 2024-12-09T08:21:03,674 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,674 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,674 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:03,676 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d7b82a71238df1b2c2d653f64b9b46b 2024-12-09T08:21:03,678 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/8d7b82a71238df1b2c2d653f64b9b46b/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T08:21:03,679 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d7b82a71238df1b2c2d653f64b9b46b; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59429999, jitterRate=-0.11442400515079498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:03,680 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d7b82a71238df1b2c2d653f64b9b46b: Writing region info on filesystem at 1733732463614Initializing all the Stores at 1733732463615 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463615Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463615Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463615Obtaining lock to block concurrent updates at 1733732463641 (+26 ms)Preparing flush snapshotting stores in 8d7b82a71238df1b2c2d653f64b9b46b at 1733732463641Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733732463641Flushing stores of testReplayEditsWrittenViaHRegion,,1733732463355.8d7b82a71238df1b2c2d653f64b9b46b. at 1733732463641Flushing 8d7b82a71238df1b2c2d653f64b9b46b/b: creating writer at 1733732463641Flushing 8d7b82a71238df1b2c2d653f64b9b46b/b: appending metadata at 1733732463653 (+12 ms)Flushing 8d7b82a71238df1b2c2d653f64b9b46b/b: closing flushed file at 1733732463653Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74409c01: reopening flushed file at 1733732463666 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8d7b82a71238df1b2c2d653f64b9b46b in 32ms, sequenceid=32, compaction requested=false; wal=null at 1733732463672 (+6 ms)Cleaning up temporary data from old regions at 1733732463674 (+2 ms)Region opened successfully at 1733732463680 (+6 ms) 2024-12-09T08:21:03,702 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=420 (was 413) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:41828 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:56082 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:49798 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:34880 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:54112 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:57482 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1123 (was 1049) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 264), ProcessCount=11 (was 11), AvailableMemoryMB=4037 (was 4053) 2024-12-09T08:21:03,702 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1123 is superior to 1024 2024-12-09T08:21:03,715 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1123, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=4036 2024-12-09T08:21:03,715 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1123 is superior to 1024 2024-12-09T08:21:03,735 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:03,738 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:03,738 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:21:03,741 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19410411, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-19410411, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:03,757 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19410411/hregion-19410411.1733732463741, exclude list is [], retry=0 2024-12-09T08:21:03,760 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:03,760 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:03,760 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:03,764 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19410411/hregion-19410411.1733732463741 2024-12-09T08:21:03,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:21:03,765 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a8da9a34d4ac525c894d5020acd52945, NAME => 'testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:21:03,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741902_1080 (size=68) 2024-12-09T08:21:03,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741902_1080 (size=68) 2024-12-09T08:21:03,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741902_1080 (size=68) 2024-12-09T08:21:03,779 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:03,780 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,781 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName a 2024-12-09T08:21:03,781 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,782 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,782 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,783 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName b 2024-12-09T08:21:03,783 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,784 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,784 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,785 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName c 2024-12-09T08:21:03,785 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,786 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,786 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,786 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,787 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,788 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,788 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,788 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:03,790 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,793 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:21:03,794 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8da9a34d4ac525c894d5020acd52945; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71095726, jitterRate=0.059408873319625854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:03,794 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8da9a34d4ac525c894d5020acd52945: Writing region info on filesystem at 1733732463779Initializing all the Stores at 1733732463779Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463780 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463780Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463780Cleaning up temporary data from old regions at 1733732463788 (+8 ms)Region opened successfully at 1733732463794 (+6 ms) 2024-12-09T08:21:03,794 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a8da9a34d4ac525c894d5020acd52945, disabling compactions & flushes 2024-12-09T08:21:03,794 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:03,795 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:03,795 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. after waiting 0 ms 2024-12-09T08:21:03,795 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:03,795 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:03,795 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a8da9a34d4ac525c894d5020acd52945: Waiting for close lock at 1733732463794Disabling compacts and flushes for region at 1733732463794Disabling writes for close at 1733732463795 (+1 ms)Writing region close event to WAL at 1733732463795Closed at 1733732463795 2024-12-09T08:21:03,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741901_1079 (size=93) 2024-12-09T08:21:03,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741901_1079 (size=93) 2024-12-09T08:21:03,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741901_1079 (size=93) 2024-12-09T08:21:03,802 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:03,802 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19410411:(num 1733732463741) 2024-12-09T08:21:03,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:03,804 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:03,815 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804, exclude list is [], retry=0 2024-12-09T08:21:03,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:03,818 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:03,819 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:03,820 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 2024-12-09T08:21:03,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:21:03,889 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a8da9a34d4ac525c894d5020acd52945, NAME => 'testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:03,891 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,891 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:03,891 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,891 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,893 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,894 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName a 2024-12-09T08:21:03,894 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,895 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,895 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,896 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName b 2024-12-09T08:21:03,896 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,896 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,896 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,897 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName c 2024-12-09T08:21:03,897 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:03,897 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:03,898 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,898 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,899 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,900 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,900 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,901 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:03,902 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,903 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8da9a34d4ac525c894d5020acd52945; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62558160, jitterRate=-0.0678107738494873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:03,903 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:03,903 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8da9a34d4ac525c894d5020acd52945: Running coprocessor pre-open hook at 1733732463892Writing region info on filesystem at 1733732463892Initializing all the Stores at 1733732463892Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463893 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463893Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732463893Cleaning up temporary data from old regions at 1733732463900 (+7 ms)Running coprocessor post-open hooks at 1733732463903 (+3 ms)Region opened successfully at 1733732463903 2024-12-09T08:21:03,915 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a8da9a34d4ac525c894d5020acd52945 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T08:21:03,916 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:04,089 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T08:21:04,089 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:04,090 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T08:21:04,091 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:04,092 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T08:21:04,092 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:04,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T08:21:04,094 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:04,095 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T08:21:04,095 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:04,917 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:05,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:21:05,918 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:06,919 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:07,920 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:08,922 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:09,922 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:10,924 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:11,925 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:12,904 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:21:12,926 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a8da9a34d4ac525c894d5020acd52945/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:12,927 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a8da9a34d4ac525c894d5020acd52945: 2024-12-09T08:21:12,928 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:12,940 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a8da9a34d4ac525c894d5020acd52945: 2024-12-09T08:21:12,940 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T08:21:12,940 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a8da9a34d4ac525c894d5020acd52945, disabling compactions & flushes 2024-12-09T08:21:12,940 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:12,940 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:12,940 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. after waiting 0 ms 2024-12-09T08:21:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:12,941 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:12,941 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. 2024-12-09T08:21:12,941 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a8da9a34d4ac525c894d5020acd52945: Waiting for close lock at 1733732472940Running coprocessor pre-close hooks at 1733732472940Disabling compacts and flushes for region at 1733732472940Disabling writes for close at 1733732472940Writing region close event to WAL at 1733732472941 (+1 ms)Running coprocessor post-close hooks at 1733732472941Closed at 1733732472941 2024-12-09T08:21:12,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741903_1081 (size=2398) 2024-12-09T08:21:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741903_1081 (size=2398) 2024-12-09T08:21:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741903_1081 (size=2398) 2024-12-09T08:21:12,962 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804, size=2.3 K (2398bytes) 2024-12-09T08:21:12,962 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 2024-12-09T08:21:12,962 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 after 0ms 2024-12-09T08:21:12,965 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:12,965 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 took 4ms 2024-12-09T08:21:12,967 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 so closing down 2024-12-09T08:21:12,967 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:12,968 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733732463804.temp 2024-12-09T08:21:12,970 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000004-wal.1733732463804.temp 2024-12-09T08:21:12,970 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741904_1082 (size=1672) 2024-12-09T08:21:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741904_1082 (size=1672) 2024-12-09T08:21:12,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741904_1082 (size=1672) 2024-12-09T08:21:12,979 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000004-wal.1733732463804.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T08:21:12,980 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000004-wal.1733732463804.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026 2024-12-09T08:21:12,981 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-09T08:21:12,981 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804, journal: Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804, size=2.3 K (2398bytes) at 1733732472962Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 so closing down at 1733732472967 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000004-wal.1733732463804.temp at 1733732472970 (+3 ms)3 split writer threads finished at 1733732472970Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000004-wal.1733732463804.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733732472979 (+9 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000004-wal.1733732463804.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026 at 1733732472980 (+1 ms)Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733732472981 (+1 ms) 2024-12-09T08:21:12,983 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732463804 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732463804 2024-12-09T08:21:12,984 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026 2024-12-09T08:21:12,984 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:12,986 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:13,002 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732472986, exclude list is [], retry=0 2024-12-09T08:21:13,004 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:13,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:13,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:13,006 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732472986 2024-12-09T08:21:13,006 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:21:13,007 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a8da9a34d4ac525c894d5020acd52945, NAME => 'testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:13,007 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,007 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,007 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,007 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,009 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,010 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName a 2024-12-09T08:21:13,011 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,011 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,011 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,012 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName b 2024-12-09T08:21:13,012 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,013 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,013 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,014 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8da9a34d4ac525c894d5020acd52945 columnFamilyName c 2024-12-09T08:21:13,014 DEBUG [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,015 INFO [StoreOpener-a8da9a34d4ac525c894d5020acd52945-1 {}] regionserver.HStore(327): Store=a8da9a34d4ac525c894d5020acd52945/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,015 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,015 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,017 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,017 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026 2024-12-09T08:21:13,020 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:13,021 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026 2024-12-09T08:21:13,022 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a8da9a34d4ac525c894d5020acd52945 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T08:21:13,034 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/a/f8c61df9973a49659bda288ebea8548b is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733732472932/Put/seqid=0 2024-12-09T08:21:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741906_1084 (size=5523) 2024-12-09T08:21:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741906_1084 (size=5523) 2024-12-09T08:21:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741906_1084 (size=5523) 2024-12-09T08:21:13,041 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/a/f8c61df9973a49659bda288ebea8548b 2024-12-09T08:21:13,058 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/b/2e7ad90e30794124835543081e7bd513 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733732472928/Put/seqid=0 2024-12-09T08:21:13,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741907_1085 (size=5524) 2024-12-09T08:21:13,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741907_1085 (size=5524) 2024-12-09T08:21:13,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741907_1085 (size=5524) 2024-12-09T08:21:13,067 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/b/2e7ad90e30794124835543081e7bd513 2024-12-09T08:21:13,084 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/c/c671d681d4524c0b824832242d591e22 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733732472930/Put/seqid=0 2024-12-09T08:21:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741908_1086 (size=5457) 2024-12-09T08:21:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741908_1086 (size=5457) 2024-12-09T08:21:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741908_1086 (size=5457) 2024-12-09T08:21:13,090 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/c/c671d681d4524c0b824832242d591e22 2024-12-09T08:21:13,095 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/a/f8c61df9973a49659bda288ebea8548b as hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/a/f8c61df9973a49659bda288ebea8548b 2024-12-09T08:21:13,101 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/a/f8c61df9973a49659bda288ebea8548b, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T08:21:13,102 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/b/2e7ad90e30794124835543081e7bd513 as hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/b/2e7ad90e30794124835543081e7bd513 2024-12-09T08:21:13,108 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/b/2e7ad90e30794124835543081e7bd513, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T08:21:13,109 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/.tmp/c/c671d681d4524c0b824832242d591e22 as hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/c/c671d681d4524c0b824832242d591e22 2024-12-09T08:21:13,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/c/c671d681d4524c0b824832242d591e22, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T08:21:13,114 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for a8da9a34d4ac525c894d5020acd52945 in 92ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T08:21:13,115 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/0000000000000000026 2024-12-09T08:21:13,116 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,116 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,116 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:13,118 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,120 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsAfterAbortingFlush/a8da9a34d4ac525c894d5020acd52945/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T08:21:13,121 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8da9a34d4ac525c894d5020acd52945; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70688417, jitterRate=0.05333949625492096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:13,121 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a8da9a34d4ac525c894d5020acd52945 2024-12-09T08:21:13,122 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8da9a34d4ac525c894d5020acd52945: Running coprocessor pre-open hook at 1733732473008Writing region info on filesystem at 1733732473008Initializing all the Stores at 1733732473008Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473008Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473009 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473009Obtaining lock to block concurrent updates at 1733732473022 (+13 ms)Preparing flush snapshotting stores in a8da9a34d4ac525c894d5020acd52945 at 1733732473022Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733732473022Flushing stores of testReplayEditsAfterAbortingFlush,,1733732463736.a8da9a34d4ac525c894d5020acd52945. at 1733732473022Flushing a8da9a34d4ac525c894d5020acd52945/a: creating writer at 1733732473022Flushing a8da9a34d4ac525c894d5020acd52945/a: appending metadata at 1733732473034 (+12 ms)Flushing a8da9a34d4ac525c894d5020acd52945/a: closing flushed file at 1733732473034Flushing a8da9a34d4ac525c894d5020acd52945/b: creating writer at 1733732473046 (+12 ms)Flushing a8da9a34d4ac525c894d5020acd52945/b: appending metadata at 1733732473058 (+12 ms)Flushing a8da9a34d4ac525c894d5020acd52945/b: closing flushed file at 1733732473058Flushing a8da9a34d4ac525c894d5020acd52945/c: creating writer at 1733732473072 (+14 ms)Flushing a8da9a34d4ac525c894d5020acd52945/c: appending metadata at 1733732473084 (+12 ms)Flushing a8da9a34d4ac525c894d5020acd52945/c: closing flushed file at 1733732473084Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7228b03e: reopening flushed file at 1733732473094 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c67c3b: reopening flushed file at 1733732473101 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f4866be: reopening flushed file at 1733732473108 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for a8da9a34d4ac525c894d5020acd52945 in 92ms, sequenceid=26, compaction requested=false; wal=null at 1733732473114 (+6 ms)Cleaning up temporary data from old regions at 1733732473116 (+2 ms)Running coprocessor post-open hooks at 1733732473121 (+5 ms)Region opened successfully at 1733732473122 (+1 ms) 2024-12-09T08:21:13,141 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=418 (was 420), OpenFileDescriptor=1183 (was 1123) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 264), ProcessCount=11 (was 11), AvailableMemoryMB=4017 (was 4036) 2024-12-09T08:21:13,141 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-09T08:21:13,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=4016 2024-12-09T08:21:13,153 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-09T08:21:13,166 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:13,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:13,168 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:21:13,170 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68005459, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-68005459, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:13,182 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68005459/hregion-68005459.1733732473171, exclude list is [], retry=0 2024-12-09T08:21:13,202 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:13,202 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:13,202 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:13,204 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68005459/hregion-68005459.1733732473171 2024-12-09T08:21:13,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:21:13,205 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 880cf842c8c35f4e92fe635ccd895182, NAME => 'testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:21:13,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741910_1088 (size=61) 2024-12-09T08:21:13,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741910_1088 (size=61) 2024-12-09T08:21:13,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741910_1088 (size=61) 2024-12-09T08:21:13,216 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,217 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,219 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880cf842c8c35f4e92fe635ccd895182 columnFamilyName a 2024-12-09T08:21:13,219 DEBUG [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,219 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(327): Store=880cf842c8c35f4e92fe635ccd895182/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,220 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,221 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,221 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,221 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,223 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,226 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:21:13,226 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 880cf842c8c35f4e92fe635ccd895182; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63696594, jitterRate=-0.05084678530693054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:21:13,227 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 880cf842c8c35f4e92fe635ccd895182: Writing region info on filesystem at 1733732473216Initializing all the Stores at 1733732473217 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473217Cleaning up temporary data from old regions at 1733732473221 (+4 ms)Region opened successfully at 1733732473227 (+6 ms) 2024-12-09T08:21:13,227 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 880cf842c8c35f4e92fe635ccd895182, disabling compactions & flushes 2024-12-09T08:21:13,227 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,227 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,227 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. after waiting 0 ms 2024-12-09T08:21:13,227 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,228 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,228 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 880cf842c8c35f4e92fe635ccd895182: Waiting for close lock at 1733732473227Disabling compacts and flushes for region at 1733732473227Disabling writes for close at 1733732473227Writing region close event to WAL at 1733732473228 (+1 ms)Closed at 1733732473228 2024-12-09T08:21:13,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741909_1087 (size=93) 2024-12-09T08:21:13,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741909_1087 (size=93) 2024-12-09T08:21:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741909_1087 (size=93) 2024-12-09T08:21:13,233 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:13,233 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68005459:(num 1733732473171) 2024-12-09T08:21:13,233 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:13,234 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:13,245 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234, exclude list is [], retry=0 2024-12-09T08:21:13,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:13,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:13,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:13,249 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 2024-12-09T08:21:13,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:21:13,249 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 880cf842c8c35f4e92fe635ccd895182, NAME => 'testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:13,249 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,250 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,250 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,252 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,253 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880cf842c8c35f4e92fe635ccd895182 columnFamilyName a 2024-12-09T08:21:13,253 DEBUG [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,254 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(327): Store=880cf842c8c35f4e92fe635ccd895182/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,254 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,255 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,256 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,256 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,256 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,258 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,259 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 880cf842c8c35f4e92fe635ccd895182; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68199797, jitterRate=0.016256168484687805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:21:13,259 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 880cf842c8c35f4e92fe635ccd895182: Writing region info on filesystem at 1733732473250Initializing all the Stores at 1733732473250Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473250Cleaning up temporary data from old regions at 1733732473256 (+6 ms)Region opened successfully at 1733732473259 (+3 ms) 2024-12-09T08:21:13,267 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 880cf842c8c35f4e92fe635ccd895182, disabling compactions & flushes 2024-12-09T08:21:13,267 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,267 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,267 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. after waiting 0 ms 2024-12-09T08:21:13,267 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,268 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,268 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. 2024-12-09T08:21:13,268 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 880cf842c8c35f4e92fe635ccd895182: Waiting for close lock at 1733732473267Disabling compacts and flushes for region at 1733732473267Disabling writes for close at 1733732473267Writing region close event to WAL at 1733732473268 (+1 ms)Closed at 1733732473268 2024-12-09T08:21:13,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741911_1089 (size=838) 2024-12-09T08:21:13,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741911_1089 (size=838) 2024-12-09T08:21:13,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741911_1089 (size=838) 2024-12-09T08:21:13,283 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234, size=838 (838bytes) 2024-12-09T08:21:13,283 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 2024-12-09T08:21:13,283 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 after 0ms 2024-12-09T08:21:13,285 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:13,285 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 took 2ms 2024-12-09T08:21:13,287 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 so closing down 2024-12-09T08:21:13,287 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:13,288 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732473234.temp 2024-12-09T08:21:13,290 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000003-wal.1733732473234.temp 2024-12-09T08:21:13,290 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:13,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741912_1090 (size=838) 2024-12-09T08:21:13,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741912_1090 (size=838) 2024-12-09T08:21:13,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741912_1090 (size=838) 2024-12-09T08:21:13,297 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000003-wal.1733732473234.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T08:21:13,298 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000003-wal.1733732473234.temp to hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012 2024-12-09T08:21:13,298 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234, size=838, length=838, corrupted=false, cancelled=false 2024-12-09T08:21:13,298 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234, journal: Splitting hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234, size=838 (838bytes) at 1733732473283Finishing writing output for hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 so closing down at 1733732473287 (+4 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000003-wal.1733732473234.temp at 1733732473290 (+3 ms)3 split writer threads finished at 1733732473290Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000003-wal.1733732473234.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733732473297 (+7 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000003-wal.1733732473234.temp to hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012 at 1733732473298 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234, size=838, length=838, corrupted=false, cancelled=false at 1733732473298 2024-12-09T08:21:13,300 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473234 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732473234 2024-12-09T08:21:13,300 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012 2024-12-09T08:21:13,303 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:13,571 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:13,574 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:13,584 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473574, exclude list is [], retry=0 2024-12-09T08:21:13,587 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:13,587 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:13,587 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:13,589 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473574 2024-12-09T08:21:13,589 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:21:13,589 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 880cf842c8c35f4e92fe635ccd895182, NAME => 'testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:13,589 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,589 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,590 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,592 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,594 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880cf842c8c35f4e92fe635ccd895182 columnFamilyName a 2024-12-09T08:21:13,594 DEBUG [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,595 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(327): Store=880cf842c8c35f4e92fe635ccd895182/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,595 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,596 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,598 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012 2024-12-09T08:21:13,599 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:13,600 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012 2024-12-09T08:21:13,600 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 880cf842c8c35f4e92fe635ccd895182 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T08:21:13,616 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/.tmp/a/3a7fd46f61d542a5a1e3105bca6012aa is 79, key is testDatalossWhenInputError/a:x0/1733732473259/Put/seqid=0 2024-12-09T08:21:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741914_1092 (size=5808) 2024-12-09T08:21:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741914_1092 (size=5808) 2024-12-09T08:21:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741914_1092 (size=5808) 2024-12-09T08:21:13,623 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/.tmp/a/3a7fd46f61d542a5a1e3105bca6012aa 2024-12-09T08:21:13,630 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/.tmp/a/3a7fd46f61d542a5a1e3105bca6012aa as hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/a/3a7fd46f61d542a5a1e3105bca6012aa 2024-12-09T08:21:13,638 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/a/3a7fd46f61d542a5a1e3105bca6012aa, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T08:21:13,639 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 880cf842c8c35f4e92fe635ccd895182 in 38ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T08:21:13,639 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/0000000000000000012 2024-12-09T08:21:13,640 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,640 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,643 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,645 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T08:21:13,646 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 880cf842c8c35f4e92fe635ccd895182; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67619300, jitterRate=0.007606089115142822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:21:13,646 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 880cf842c8c35f4e92fe635ccd895182: Writing region info on filesystem at 1733732473590Initializing all the Stores at 1733732473592 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473592Obtaining lock to block concurrent updates at 1733732473600 (+8 ms)Preparing flush snapshotting stores in 880cf842c8c35f4e92fe635ccd895182 at 1733732473600Finished memstore snapshotting testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733732473601 (+1 ms)Flushing stores of testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182. at 1733732473601Flushing 880cf842c8c35f4e92fe635ccd895182/a: creating writer at 1733732473601Flushing 880cf842c8c35f4e92fe635ccd895182/a: appending metadata at 1733732473616 (+15 ms)Flushing 880cf842c8c35f4e92fe635ccd895182/a: closing flushed file at 1733732473616Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e6a8e04: reopening flushed file at 1733732473629 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 880cf842c8c35f4e92fe635ccd895182 in 38ms, sequenceid=12, compaction requested=false; wal=null at 1733732473639 (+10 ms)Cleaning up temporary data from old regions at 1733732473640 (+1 ms)Region opened successfully at 1733732473646 (+6 ms) 2024-12-09T08:21:13,649 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 880cf842c8c35f4e92fe635ccd895182, NAME => 'testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:13,650 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732473166.880cf842c8c35f4e92fe635ccd895182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,650 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,650 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,651 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,652 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880cf842c8c35f4e92fe635ccd895182 columnFamilyName a 2024-12-09T08:21:13,652 DEBUG [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,657 DEBUG [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/a/3a7fd46f61d542a5a1e3105bca6012aa 2024-12-09T08:21:13,657 INFO [StoreOpener-880cf842c8c35f4e92fe635ccd895182-1 {}] regionserver.HStore(327): Store=880cf842c8c35f4e92fe635ccd895182/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,657 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,658 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,659 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,660 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,660 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,661 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 880cf842c8c35f4e92fe635ccd895182 2024-12-09T08:21:13,663 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testDatalossWhenInputError/880cf842c8c35f4e92fe635ccd895182/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T08:21:13,664 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 880cf842c8c35f4e92fe635ccd895182; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73778218, jitterRate=0.09938111901283264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:21:13,664 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 880cf842c8c35f4e92fe635ccd895182: Writing region info on filesystem at 1733732473650Initializing all the Stores at 1733732473651 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473651Cleaning up temporary data from old regions at 1733732473660 (+9 ms)Region opened successfully at 1733732473664 (+4 ms) 2024-12-09T08:21:13,678 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:48510 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:58310 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:58234 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:50654 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-361248767-172.17.0.2-1733732439363:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:50732 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=4007 (was 4016) 2024-12-09T08:21:13,678 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T08:21:13,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=4006 2024-12-09T08:21:13,688 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T08:21:13,701 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:13,703 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:13,704 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:21:13,706 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78056472, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-78056472, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:13,717 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78056472/hregion-78056472.1733732473706, exclude list is [], retry=0 2024-12-09T08:21:13,719 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:13,719 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:13,719 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:13,721 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78056472/hregion-78056472.1733732473706 2024-12-09T08:21:13,721 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:21:13,721 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 45e3aa44316f00d7dd39ce71151299f3, NAME => 'testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:21:13,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741916_1094 (size=63) 2024-12-09T08:21:13,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741916_1094 (size=63) 2024-12-09T08:21:13,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741916_1094 (size=63) 2024-12-09T08:21:13,728 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,729 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,730 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName a 2024-12-09T08:21:13,730 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,731 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,731 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,732 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName b 2024-12-09T08:21:13,732 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,732 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,732 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,734 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName c 2024-12-09T08:21:13,734 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,735 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,735 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,736 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,736 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,738 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:13,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,741 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:21:13,741 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45e3aa44316f00d7dd39ce71151299f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73037577, jitterRate=0.08834470808506012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:13,742 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45e3aa44316f00d7dd39ce71151299f3: Writing region info on filesystem at 1733732473728Initializing all the Stores at 1733732473729 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473729Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473729Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473729Cleaning up temporary data from old regions at 1733732473737 (+8 ms)Region opened successfully at 1733732473742 (+5 ms) 2024-12-09T08:21:13,742 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45e3aa44316f00d7dd39ce71151299f3, disabling compactions & flushes 2024-12-09T08:21:13,742 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:13,742 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:13,742 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. after waiting 0 ms 2024-12-09T08:21:13,742 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:13,742 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:13,742 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45e3aa44316f00d7dd39ce71151299f3: Waiting for close lock at 1733732473742Disabling compacts and flushes for region at 1733732473742Disabling writes for close at 1733732473742Writing region close event to WAL at 1733732473742Closed at 1733732473742 2024-12-09T08:21:13,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741915_1093 (size=93) 2024-12-09T08:21:13,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741915_1093 (size=93) 2024-12-09T08:21:13,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741915_1093 (size=93) 2024-12-09T08:21:13,747 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:13,747 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78056472:(num 1733732473706) 2024-12-09T08:21:13,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:13,749 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:13,763 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749, exclude list is [], retry=0 2024-12-09T08:21:13,765 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:13,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:13,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:13,767 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 2024-12-09T08:21:13,767 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:21:13,768 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e3aa44316f00d7dd39ce71151299f3, NAME => 'testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:13,768 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:13,768 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,768 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,770 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,771 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName a 2024-12-09T08:21:13,771 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,772 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,772 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,773 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName b 2024-12-09T08:21:13,773 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,773 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,773 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,774 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName c 2024-12-09T08:21:13,774 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:13,775 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:13,775 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,776 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,778 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,778 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,779 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:13,780 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:13,781 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45e3aa44316f00d7dd39ce71151299f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73914028, jitterRate=0.10140484571456909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:13,782 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45e3aa44316f00d7dd39ce71151299f3: Writing region info on filesystem at 1733732473768Initializing all the Stores at 1733732473769 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473769Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473770 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732473770Cleaning up temporary data from old regions at 1733732473778 (+8 ms)Region opened successfully at 1733732473782 (+4 ms) 2024-12-09T08:21:13,787 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733732473786/Put/seqid=0 2024-12-09T08:21:13,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741918_1096 (size=4875) 2024-12-09T08:21:13,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741918_1096 (size=4875) 2024-12-09T08:21:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741918_1096 (size=4875) 2024-12-09T08:21:13,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733732473800/Put/seqid=0 2024-12-09T08:21:13,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741919_1097 (size=4875) 2024-12-09T08:21:13,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741919_1097 (size=4875) 2024-12-09T08:21:13,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741919_1097 (size=4875) 2024-12-09T08:21:13,810 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733732473810/Put/seqid=0 2024-12-09T08:21:13,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741920_1098 (size=4875) 2024-12-09T08:21:13,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741920_1098 (size=4875) 2024-12-09T08:21:13,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741920_1098 (size=4875) 2024-12-09T08:21:13,816 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 45e3aa44316f00d7dd39ce71151299f3/a 2024-12-09T08:21:13,820 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T08:21:13,820 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:21:13,820 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 45e3aa44316f00d7dd39ce71151299f3/a 2024-12-09T08:21:13,824 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T08:21:13,824 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:21:13,824 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 45e3aa44316f00d7dd39ce71151299f3/a 2024-12-09T08:21:13,827 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T08:21:13,828 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:21:13,828 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45e3aa44316f00d7dd39ce71151299f3 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T08:21:13,844 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/.tmp/a/eea88374f76c4aed9ad6faeeddd8bfcf is 55, key is testCompactedBulkLoadedFiles/a:a/1733732473782/Put/seqid=0 2024-12-09T08:21:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741921_1099 (size=5107) 2024-12-09T08:21:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741921_1099 (size=5107) 2024-12-09T08:21:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741921_1099 (size=5107) 2024-12-09T08:21:13,851 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/.tmp/a/eea88374f76c4aed9ad6faeeddd8bfcf 2024-12-09T08:21:13,856 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/.tmp/a/eea88374f76c4aed9ad6faeeddd8bfcf as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf 2024-12-09T08:21:13,861 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T08:21:13,862 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 45e3aa44316f00d7dd39ce71151299f3 in 34ms, sequenceid=4, compaction requested=false 2024-12-09T08:21:13,862 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 45e3aa44316f00d7dd39ce71151299f3: 2024-12-09T08:21:13,864 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ 2024-12-09T08:21:13,865 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ 2024-12-09T08:21:13,866 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ 2024-12-09T08:21:13,867 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile0 into 45e3aa44316f00d7dd39ce71151299f3/a as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ - updating store file list. 2024-12-09T08:21:13,872 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 539ae220532a4183bba1cf008beb837e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:21:13,872 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ into 45e3aa44316f00d7dd39ce71151299f3/a 2024-12-09T08:21:13,872 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile0 into 45e3aa44316f00d7dd39ce71151299f3/a (new location: hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_) 2024-12-09T08:21:13,873 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile1 into 45e3aa44316f00d7dd39ce71151299f3/a as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ - updating store file list. 2024-12-09T08:21:13,879 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 75b0c691135843a0aa89c668713b7657_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:21:13,879 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ into 45e3aa44316f00d7dd39ce71151299f3/a 2024-12-09T08:21:13,879 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile1 into 45e3aa44316f00d7dd39ce71151299f3/a (new location: hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_) 2024-12-09T08:21:13,880 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile2 into 45e3aa44316f00d7dd39ce71151299f3/a as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ - updating store file list. 2024-12-09T08:21:13,884 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:21:13,884 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ into 45e3aa44316f00d7dd39ce71151299f3/a 2024-12-09T08:21:13,884 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40523/hbase/testCompactedBulkLoadedFiles/hfile2 into 45e3aa44316f00d7dd39ce71151299f3/a (new location: hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_) 2024-12-09T08:21:13,891 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:21:13,892 DEBUG [Time-limited test {}] regionserver.HStore(1541): 45e3aa44316f00d7dd39ce71151299f3/a is initiating major compaction (all files) 2024-12-09T08:21:13,892 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 45e3aa44316f00d7dd39ce71151299f3/a in testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:13,892 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_] into tmpdir=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/.tmp, totalSize=19.3 K 2024-12-09T08:21:13,892 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eea88374f76c4aed9ad6faeeddd8bfcf, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733732473782 2024-12-09T08:21:13,893 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 539ae220532a4183bba1cf008beb837e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T08:21:13,893 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 75b0c691135843a0aa89c668713b7657_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T08:21:13,894 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T08:21:13,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/.tmp/a/7987716d5a5d4e5bab111e19ae320e78 is 55, key is testCompactedBulkLoadedFiles/a:a/1733732473782/Put/seqid=0 2024-12-09T08:21:13,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741922_1100 (size=6154) 2024-12-09T08:21:13,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741922_1100 (size=6154) 2024-12-09T08:21:13,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741922_1100 (size=6154) 2024-12-09T08:21:13,923 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/.tmp/a/7987716d5a5d4e5bab111e19ae320e78 as hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/7987716d5a5d4e5bab111e19ae320e78 2024-12-09T08:21:13,929 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 45e3aa44316f00d7dd39ce71151299f3/a of 45e3aa44316f00d7dd39ce71151299f3 into 7987716d5a5d4e5bab111e19ae320e78(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:21:13,929 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 45e3aa44316f00d7dd39ce71151299f3: 2024-12-09T08:21:13,929 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T08:21:13,930 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T08:21:13,960 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749, size=0 (0bytes) 2024-12-09T08:21:13,960 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 might be still open, length is 0 2024-12-09T08:21:13,960 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 2024-12-09T08:21:13,960 WARN [IPC Server handler 4 on default port 40523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T08:21:13,961 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 after 1ms 2024-12-09T08:21:14,089 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T08:21:14,089 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:14,089 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T08:21:14,089 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T08:21:15,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:58356 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38773:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58356 dst: /127.0.0.1:38773 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38773 remote=/127.0.0.1:58356]. Total timeout mills is 60000, 58066 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:15,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:50766 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50766 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:15,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:48558 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41795:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48558 dst: /127.0.0.1:41795 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:15,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741917_1101 (size=1368) 2024-12-09T08:21:15,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741917_1101 (size=1368) 2024-12-09T08:21:17,961 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 after 4001ms 2024-12-09T08:21:17,964 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:17,964 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 took 4004ms 2024-12-09T08:21:17,968 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749; continuing. 2024-12-09T08:21:17,968 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 so closing down 2024-12-09T08:21:17,969 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:17,970 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732473749.temp 2024-12-09T08:21:17,971 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000003-wal.1733732473749.temp 2024-12-09T08:21:17,971 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741923_1102 (size=635) 2024-12-09T08:21:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741923_1102 (size=635) 2024-12-09T08:21:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741923_1102 (size=635) 2024-12-09T08:21:17,979 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000003-wal.1733732473749.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T08:21:17,980 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000003-wal.1733732473749.temp to hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008 2024-12-09T08:21:17,980 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:21:17,980 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749, journal: Splitting hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749, size=0 (0bytes) at 1733732473960Finishing writing output for hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 so closing down at 1733732477969 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000003-wal.1733732473749.temp at 1733732477971 (+2 ms)3 split writer threads finished at 1733732477971Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000003-wal.1733732473749.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733732477979 (+8 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000003-wal.1733732473749.temp to hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008 at 1733732477980 (+1 ms)Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749, size=0, length=0, corrupted=false, cancelled=false at 1733732477980 2024-12-09T08:21:17,981 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732473749 2024-12-09T08:21:17,982 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008 2024-12-09T08:21:17,982 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:17,984 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:17,995 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732477984, exclude list is [], retry=0 2024-12-09T08:21:17,997 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:17,997 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:17,998 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:17,999 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732477984 2024-12-09T08:21:17,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:21:17,999 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e3aa44316f00d7dd39ce71151299f3, NAME => 'testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:18,000 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:18,000 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,000 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,001 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,002 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName a 2024-12-09T08:21:18,002 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,008 DEBUG [StoreFileOpener-45e3aa44316f00d7dd39ce71151299f3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:21:18,008 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ 2024-12-09T08:21:18,012 DEBUG [StoreFileOpener-45e3aa44316f00d7dd39ce71151299f3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 539ae220532a4183bba1cf008beb837e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:21:18,012 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ 2024-12-09T08:21:18,016 DEBUG [StoreFileOpener-45e3aa44316f00d7dd39ce71151299f3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 75b0c691135843a0aa89c668713b7657_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:21:18,016 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ 2024-12-09T08:21:18,020 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/7987716d5a5d4e5bab111e19ae320e78 2024-12-09T08:21:18,025 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf 2024-12-09T08:21:18,025 WARN [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@12033c4f 2024-12-09T08:21:18,025 WARN [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@12033c4f 2024-12-09T08:21:18,025 WARN [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@12033c4f 2024-12-09T08:21:18,026 WARN [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@12033c4f 2024-12-09T08:21:18,026 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf] to archive 2024-12-09T08:21:18,027 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:21:18,028 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ to hdfs://localhost:40523/hbase/archive/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_ 2024-12-09T08:21:18,030 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ to hdfs://localhost:40523/hbase/archive/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/539ae220532a4183bba1cf008beb837e_SeqId_4_ 2024-12-09T08:21:18,031 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ to hdfs://localhost:40523/hbase/archive/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/75b0c691135843a0aa89c668713b7657_SeqId_4_ 2024-12-09T08:21:18,032 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf to hdfs://localhost:40523/hbase/archive/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/a/eea88374f76c4aed9ad6faeeddd8bfcf 2024-12-09T08:21:18,032 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,032 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,033 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName b 2024-12-09T08:21:18,033 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,033 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,034 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,034 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e3aa44316f00d7dd39ce71151299f3 columnFamilyName c 2024-12-09T08:21:18,034 DEBUG [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,035 INFO [StoreOpener-45e3aa44316f00d7dd39ce71151299f3-1 {}] regionserver.HStore(327): Store=45e3aa44316f00d7dd39ce71151299f3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,035 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,035 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,037 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008 2024-12-09T08:21:18,038 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:18,040 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 45e3aa44316f00d7dd39ce71151299f3 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "45e3aa44316f00d7dd39ce71151299f3" family_name: "a" compaction_input: "eea88374f76c4aed9ad6faeeddd8bfcf" compaction_input: "539ae220532a4183bba1cf008beb837e_SeqId_4_" compaction_input: "75b0c691135843a0aa89c668713b7657_SeqId_4_" compaction_input: "0d3d04c5b00b4f1f9bb76db7dc49c6f7_SeqId_4_" compaction_output: "7987716d5a5d4e5bab111e19ae320e78" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T08:21:18,040 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T08:21:18,040 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008 2024-12-09T08:21:18,041 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/0000000000000000008 2024-12-09T08:21:18,042 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,042 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,042 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:18,044 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45e3aa44316f00d7dd39ce71151299f3 2024-12-09T08:21:18,046 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testCompactedBulkLoadedFiles/45e3aa44316f00d7dd39ce71151299f3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T08:21:18,047 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45e3aa44316f00d7dd39ce71151299f3; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63911999, jitterRate=-0.04763700067996979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:18,047 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45e3aa44316f00d7dd39ce71151299f3: Writing region info on filesystem at 1733732478000Initializing all the Stores at 1733732478001 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478001Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478001Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478001Cleaning up temporary data from old regions at 1733732478042 (+41 ms)Region opened successfully at 1733732478047 (+5 ms) 2024-12-09T08:21:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45e3aa44316f00d7dd39ce71151299f3, disabling compactions & flushes 2024-12-09T08:21:18,050 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:18,051 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:18,051 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. after waiting 0 ms 2024-12-09T08:21:18,051 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:18,051 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733732473702.45e3aa44316f00d7dd39ce71151299f3. 2024-12-09T08:21:18,051 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45e3aa44316f00d7dd39ce71151299f3: Waiting for close lock at 1733732478050Disabling compacts and flushes for region at 1733732478050Disabling writes for close at 1733732478051 (+1 ms)Writing region close event to WAL at 1733732478051Closed at 1733732478051 2024-12-09T08:21:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741924_1103 (size=93) 2024-12-09T08:21:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741924_1103 (size=93) 2024-12-09T08:21:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741924_1103 (size=93) 2024-12-09T08:21:18,057 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:18,057 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732477984) 2024-12-09T08:21:18,069 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=441 (was 428) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-288839535_22 at /127.0.0.1:48668 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-288839535_22 at /127.0.0.1:50812 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40523 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40523 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-288839535_22 at /127.0.0.1:58416 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=4004 (was 4006) 2024-12-09T08:21:18,069 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-09T08:21:18,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=441, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=4003 2024-12-09T08:21:18,082 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-09T08:21:18,094 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:18,096 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:21:18,097 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:21:18,099 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44059456, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/hregion-44059456, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:18,111 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44059456/hregion-44059456.1733732478100, exclude list is [], retry=0 2024-12-09T08:21:18,113 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:18,114 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:18,114 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:18,116 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44059456/hregion-44059456.1733732478100 2024-12-09T08:21:18,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:21:18,117 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7bec8f4816b72fe84f3bf1daf6aade26, NAME => 'testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40523/hbase 2024-12-09T08:21:18,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741926_1105 (size=67) 2024-12-09T08:21:18,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741926_1105 (size=67) 2024-12-09T08:21:18,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741926_1105 (size=67) 2024-12-09T08:21:18,129 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:18,133 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,134 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName a 2024-12-09T08:21:18,135 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,135 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,135 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,136 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName b 2024-12-09T08:21:18,136 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,137 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,137 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,138 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName c 2024-12-09T08:21:18,138 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,138 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,138 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,139 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,139 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,140 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,140 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,141 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:18,142 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,144 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:21:18,144 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7bec8f4816b72fe84f3bf1daf6aade26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60590229, jitterRate=-0.09713523089885712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:18,144 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7bec8f4816b72fe84f3bf1daf6aade26: Writing region info on filesystem at 1733732478130Initializing all the Stores at 1733732478130Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478130Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478133 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478133Cleaning up temporary data from old regions at 1733732478140 (+7 ms)Region opened successfully at 1733732478144 (+4 ms) 2024-12-09T08:21:18,144 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7bec8f4816b72fe84f3bf1daf6aade26, disabling compactions & flushes 2024-12-09T08:21:18,144 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,144 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,145 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. after waiting 0 ms 2024-12-09T08:21:18,145 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,145 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,145 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7bec8f4816b72fe84f3bf1daf6aade26: Waiting for close lock at 1733732478144Disabling compacts and flushes for region at 1733732478144Disabling writes for close at 1733732478145 (+1 ms)Writing region close event to WAL at 1733732478145Closed at 1733732478145 2024-12-09T08:21:18,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741925_1104 (size=93) 2024-12-09T08:21:18,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741925_1104 (size=93) 2024-12-09T08:21:18,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741925_1104 (size=93) 2024-12-09T08:21:18,149 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:18,150 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44059456:(num 1733732478100) 2024-12-09T08:21:18,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:18,151 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:18,164 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151, exclude list is [], retry=0 2024-12-09T08:21:18,167 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:18,167 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:18,167 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:18,168 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 2024-12-09T08:21:18,169 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:33395:33395)] 2024-12-09T08:21:18,169 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7bec8f4816b72fe84f3bf1daf6aade26, NAME => 'testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:18,169 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:18,169 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,169 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,170 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,171 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName a 2024-12-09T08:21:18,171 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,171 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,171 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,172 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName b 2024-12-09T08:21:18,172 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,172 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,173 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,173 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName c 2024-12-09T08:21:18,173 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,174 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,174 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,177 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,177 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,177 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:18,179 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,180 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7bec8f4816b72fe84f3bf1daf6aade26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67969737, jitterRate=0.012828007340431213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:18,180 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7bec8f4816b72fe84f3bf1daf6aade26: Writing region info on filesystem at 1733732478169Initializing all the Stores at 1733732478170 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478170Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478170Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478170Cleaning up temporary data from old regions at 1733732478177 (+7 ms)Region opened successfully at 1733732478180 (+3 ms) 2024-12-09T08:21:18,187 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7bec8f4816b72fe84f3bf1daf6aade26 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T08:21:18,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/a/ccdf9d11930d42c1a348b1446e2d8324 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733732478180/Put/seqid=0 2024-12-09T08:21:18,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741928_1107 (size=5958) 2024-12-09T08:21:18,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741928_1107 (size=5958) 2024-12-09T08:21:18,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741928_1107 (size=5958) 2024-12-09T08:21:18,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/a/ccdf9d11930d42c1a348b1446e2d8324 2024-12-09T08:21:18,268 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/a/ccdf9d11930d42c1a348b1446e2d8324 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/a/ccdf9d11930d42c1a348b1446e2d8324 2024-12-09T08:21:18,273 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/a/ccdf9d11930d42c1a348b1446e2d8324, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T08:21:18,274 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7bec8f4816b72fe84f3bf1daf6aade26 in 87ms, sequenceid=13, compaction requested=false 2024-12-09T08:21:18,274 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7bec8f4816b72fe84f3bf1daf6aade26: 2024-12-09T08:21:18,290 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7bec8f4816b72fe84f3bf1daf6aade26, disabling compactions & flushes 2024-12-09T08:21:18,290 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,290 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,290 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. after waiting 0 ms 2024-12-09T08:21:18,290 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,291 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,291 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:18,291 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7bec8f4816b72fe84f3bf1daf6aade26: Waiting for close lock at 1733732478290Disabling compacts and flushes for region at 1733732478290Disabling writes for close at 1733732478290Writing region close event to WAL at 1733732478291 (+1 ms)Closed at 1733732478291 2024-12-09T08:21:18,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741927_1106 (size=2805) 2024-12-09T08:21:18,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741927_1106 (size=2805) 2024-12-09T08:21:18,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741927_1106 (size=2805) 2024-12-09T08:21:18,309 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151, size=2.7 K (2805bytes) 2024-12-09T08:21:18,309 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 2024-12-09T08:21:18,309 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 after 0ms 2024-12-09T08:21:18,311 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:18,311 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 took 3ms 2024-12-09T08:21:18,313 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 so closing down 2024-12-09T08:21:18,313 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:18,313 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732478151.temp 2024-12-09T08:21:18,314 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000003-wal.1733732478151.temp 2024-12-09T08:21:18,315 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:18,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741929_1108 (size=2312) 2024-12-09T08:21:18,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741929_1108 (size=2312) 2024-12-09T08:21:18,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741929_1108 (size=2312) 2024-12-09T08:21:18,325 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000003-wal.1733732478151.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T08:21:18,327 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000003-wal.1733732478151.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035 2024-12-09T08:21:18,327 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-09T08:21:18,327 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151, journal: Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151, size=2.7 K (2805bytes) at 1733732478309Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 so closing down at 1733732478313 (+4 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000003-wal.1733732478151.temp at 1733732478314 (+1 ms)3 split writer threads finished at 1733732478315 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000003-wal.1733732478151.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733732478325 (+10 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000003-wal.1733732478151.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035 at 1733732478327 (+2 ms)Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733732478327 2024-12-09T08:21:18,328 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478151 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732478151 2024-12-09T08:21:18,329 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035 2024-12-09T08:21:18,329 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:18,331 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:18,342 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331, exclude list is [], retry=0 2024-12-09T08:21:18,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:18,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:18,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:18,347 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 2024-12-09T08:21:18,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:45923:45923),(127.0.0.1/127.0.0.1:35705:35705)] 2024-12-09T08:21:18,347 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7bec8f4816b72fe84f3bf1daf6aade26, NAME => 'testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:21:18,347 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:18,347 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,347 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,348 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,349 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName a 2024-12-09T08:21:18,349 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,355 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/a/ccdf9d11930d42c1a348b1446e2d8324 2024-12-09T08:21:18,355 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,355 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,356 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName b 2024-12-09T08:21:18,356 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,357 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,357 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,357 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName c 2024-12-09T08:21:18,357 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:18,358 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:18,358 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,359 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,360 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,361 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035 2024-12-09T08:21:18,363 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:18,364 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035 2024-12-09T08:21:18,364 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7bec8f4816b72fe84f3bf1daf6aade26 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T08:21:18,376 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/b/1fa9588621fb4773ab09877e1cbbb440 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733732478274/Put/seqid=0 2024-12-09T08:21:18,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741931_1110 (size=5958) 2024-12-09T08:21:18,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741931_1110 (size=5958) 2024-12-09T08:21:18,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741931_1110 (size=5958) 2024-12-09T08:21:18,388 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/b/1fa9588621fb4773ab09877e1cbbb440 2024-12-09T08:21:18,409 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/c/9c9223b467c549f4be0bdb564affc7ca is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733732478281/Put/seqid=0 2024-12-09T08:21:18,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741932_1111 (size=5958) 2024-12-09T08:21:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741932_1111 (size=5958) 2024-12-09T08:21:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741932_1111 (size=5958) 2024-12-09T08:21:18,416 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/c/9c9223b467c549f4be0bdb564affc7ca 2024-12-09T08:21:18,421 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/b/1fa9588621fb4773ab09877e1cbbb440 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/b/1fa9588621fb4773ab09877e1cbbb440 2024-12-09T08:21:18,426 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/b/1fa9588621fb4773ab09877e1cbbb440, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T08:21:18,427 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/c/9c9223b467c549f4be0bdb564affc7ca as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/c/9c9223b467c549f4be0bdb564affc7ca 2024-12-09T08:21:18,432 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/c/9c9223b467c549f4be0bdb564affc7ca, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T08:21:18,432 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 7bec8f4816b72fe84f3bf1daf6aade26 in 68ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T08:21:18,433 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000035 2024-12-09T08:21:18,434 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,434 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,434 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:18,435 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:18,437 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T08:21:18,438 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7bec8f4816b72fe84f3bf1daf6aade26; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71565300, jitterRate=0.06640607118606567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:18,438 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7bec8f4816b72fe84f3bf1daf6aade26: Writing region info on filesystem at 1733732478347Initializing all the Stores at 1733732478348 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478348Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478348Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732478348Obtaining lock to block concurrent updates at 1733732478364 (+16 ms)Preparing flush snapshotting stores in 7bec8f4816b72fe84f3bf1daf6aade26 at 1733732478364Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733732478364Flushing stores of testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. at 1733732478364Flushing 7bec8f4816b72fe84f3bf1daf6aade26/b: creating writer at 1733732478364Flushing 7bec8f4816b72fe84f3bf1daf6aade26/b: appending metadata at 1733732478376 (+12 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/b: closing flushed file at 1733732478376Flushing 7bec8f4816b72fe84f3bf1daf6aade26/c: creating writer at 1733732478393 (+17 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/c: appending metadata at 1733732478409 (+16 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/c: closing flushed file at 1733732478409Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c245ac5: reopening flushed file at 1733732478420 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@437f9bf7: reopening flushed file at 1733732478427 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 7bec8f4816b72fe84f3bf1daf6aade26 in 68ms, sequenceid=35, compaction requested=false; wal=null at 1733732478432 (+5 ms)Cleaning up temporary data from old regions at 1733732478434 (+2 ms)Region opened successfully at 1733732478438 (+4 ms) 2024-12-09T08:21:18,464 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:21:18,502 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331, size=0 (0bytes) 2024-12-09T08:21:18,502 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 might be still open, length is 0 2024-12-09T08:21:18,502 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 2024-12-09T08:21:18,503 WARN [IPC Server handler 3 on default port 40523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T08:21:18,503 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 after 1ms 2024-12-09T08:21:18,862 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:48718 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41795:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48718 dst: /127.0.0.1:41795 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41795 remote=/127.0.0.1:48718]. Total timeout mills is 60000, 59603 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:18,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:50892 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50892 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:18,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_361199537_22 at /127.0.0.1:58482 [Receiving block BP-361248767-172.17.0.2-1733732439363:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38773:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58482 dst: /127.0.0.1:38773 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:18,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741930_1112 (size=2304) 2024-12-09T08:21:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741930_1112 (size=2304) 2024-12-09T08:21:18,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741930_1112 (size=2304) 2024-12-09T08:21:22,505 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 after 4002ms 2024-12-09T08:21:22,512 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:22,513 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 took 4011ms 2024-12-09T08:21:22,515 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331; continuing. 2024-12-09T08:21:22,515 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 so closing down 2024-12-09T08:21:22,515 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:21:22,517 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733732478331.temp 2024-12-09T08:21:22,519 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000037-wal.1733732478331.temp 2024-12-09T08:21:22,519 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:21:22,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741933_1113 (size=2312) 2024-12-09T08:21:22,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741933_1113 (size=2312) 2024-12-09T08:21:22,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741933_1113 (size=2312) 2024-12-09T08:21:22,528 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000037-wal.1733732478331.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T08:21:22,529 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000037-wal.1733732478331.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066 2024-12-09T08:21:22,530 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:21:22,530 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331, journal: Splitting hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331, size=0 (0bytes) at 1733732478502Finishing writing output for hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 so closing down at 1733732482515 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000037-wal.1733732478331.temp at 1733732482519 (+4 ms)3 split writer threads finished at 1733732482519Closed recovered edits writer path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000037-wal.1733732478331.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733732482528 (+9 ms)Rename recovered edits hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000037-wal.1733732478331.temp to hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066 at 1733732482529 (+1 ms)Processed 30 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331, size=0, length=0, corrupted=false, cancelled=false at 1733732482530 (+1 ms) 2024-12-09T08:21:22,531 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 to hdfs://localhost:40523/hbase/oldWALs/wal.1733732478331 2024-12-09T08:21:22,532 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066 2024-12-09T08:21:22,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:21:22,533 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40523/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094, archiveDir=hdfs://localhost:40523/hbase/oldWALs, maxLogs=32 2024-12-09T08:21:22,545 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732482534, exclude list is [], retry=0 2024-12-09T08:21:22,547 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41795,DS-cdb20101-1a35-4f28-b07d-ecf312072bf9,DISK] 2024-12-09T08:21:22,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-68eadaa8-43f5-4e5e-aad4-8a35f9641ef4,DISK] 2024-12-09T08:21:22,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38773,DS-6ff8f618-13fb-4fe5-8163-398e337dc475,DISK] 2024-12-09T08:21:22,549 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732482534 2024-12-09T08:21:22,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33395:33395),(127.0.0.1/127.0.0.1:35705:35705),(127.0.0.1/127.0.0.1:45923:45923)] 2024-12-09T08:21:22,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:21:22,551 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,551 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName a 2024-12-09T08:21:22,551 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:22,556 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/a/ccdf9d11930d42c1a348b1446e2d8324 2024-12-09T08:21:22,556 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:22,556 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,557 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName b 2024-12-09T08:21:22,557 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:22,561 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/b/1fa9588621fb4773ab09877e1cbbb440 2024-12-09T08:21:22,561 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:22,561 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,562 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bec8f4816b72fe84f3bf1daf6aade26 columnFamilyName c 2024-12-09T08:21:22,562 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:21:22,567 DEBUG [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/c/9c9223b467c549f4be0bdb564affc7ca 2024-12-09T08:21:22,567 INFO [StoreOpener-7bec8f4816b72fe84f3bf1daf6aade26-1 {}] regionserver.HStore(327): Store=7bec8f4816b72fe84f3bf1daf6aade26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:21:22,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,570 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066 2024-12-09T08:21:22,571 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T08:21:22,575 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066 2024-12-09T08:21:22,575 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7bec8f4816b72fe84f3bf1daf6aade26 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T08:21:22,588 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/a/7add08ccbcf54cf6bdcf06cee3d84989 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733732478444/Put/seqid=0 2024-12-09T08:21:22,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741935_1115 (size=5958) 2024-12-09T08:21:22,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741935_1115 (size=5958) 2024-12-09T08:21:22,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741935_1115 (size=5958) 2024-12-09T08:21:22,599 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/a/7add08ccbcf54cf6bdcf06cee3d84989 2024-12-09T08:21:22,638 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/b/61240d688a5c48218ea58a63397fda4b is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733732478451/Put/seqid=0 2024-12-09T08:21:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741936_1116 (size=5958) 2024-12-09T08:21:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741936_1116 (size=5958) 2024-12-09T08:21:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741936_1116 (size=5958) 2024-12-09T08:21:22,644 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/b/61240d688a5c48218ea58a63397fda4b 2024-12-09T08:21:22,660 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/c/851e0aeb542843f5a7c98ddc8002c17f is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733732478459/Put/seqid=0 2024-12-09T08:21:22,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741937_1117 (size=5958) 2024-12-09T08:21:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741937_1117 (size=5958) 2024-12-09T08:21:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741937_1117 (size=5958) 2024-12-09T08:21:22,665 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/c/851e0aeb542843f5a7c98ddc8002c17f 2024-12-09T08:21:22,669 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/a/7add08ccbcf54cf6bdcf06cee3d84989 as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/a/7add08ccbcf54cf6bdcf06cee3d84989 2024-12-09T08:21:22,673 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/a/7add08ccbcf54cf6bdcf06cee3d84989, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T08:21:22,674 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/b/61240d688a5c48218ea58a63397fda4b as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/b/61240d688a5c48218ea58a63397fda4b 2024-12-09T08:21:22,679 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/b/61240d688a5c48218ea58a63397fda4b, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T08:21:22,681 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/.tmp/c/851e0aeb542843f5a7c98ddc8002c17f as hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/c/851e0aeb542843f5a7c98ddc8002c17f 2024-12-09T08:21:22,685 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/c/851e0aeb542843f5a7c98ddc8002c17f, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T08:21:22,685 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 7bec8f4816b72fe84f3bf1daf6aade26 in 110ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T08:21:22,686 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/0000000000000000066 2024-12-09T08:21:22,687 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,687 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,687 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:21:22,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7bec8f4816b72fe84f3bf1daf6aade26 2024-12-09T08:21:22,690 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/hbase/data/default/testReplayEditsWrittenViaHRegion/7bec8f4816b72fe84f3bf1daf6aade26/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T08:21:22,691 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7bec8f4816b72fe84f3bf1daf6aade26; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70947385, jitterRate=0.0571984201669693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:21:22,691 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7bec8f4816b72fe84f3bf1daf6aade26: Writing region info on filesystem at 1733732482549Initializing all the Stores at 1733732482550 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732482550Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732482550Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732482550Obtaining lock to block concurrent updates at 1733732482575 (+25 ms)Preparing flush snapshotting stores in 7bec8f4816b72fe84f3bf1daf6aade26 at 1733732482575Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733732482576 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. at 1733732482576Flushing 7bec8f4816b72fe84f3bf1daf6aade26/a: creating writer at 1733732482576Flushing 7bec8f4816b72fe84f3bf1daf6aade26/a: appending metadata at 1733732482588 (+12 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/a: closing flushed file at 1733732482588Flushing 7bec8f4816b72fe84f3bf1daf6aade26/b: creating writer at 1733732482626 (+38 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/b: appending metadata at 1733732482638 (+12 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/b: closing flushed file at 1733732482638Flushing 7bec8f4816b72fe84f3bf1daf6aade26/c: creating writer at 1733732482647 (+9 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/c: appending metadata at 1733732482659 (+12 ms)Flushing 7bec8f4816b72fe84f3bf1daf6aade26/c: closing flushed file at 1733732482659Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65c90cc3: reopening flushed file at 1733732482668 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40047bc5: reopening flushed file at 1733732482674 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e083595: reopening flushed file at 1733732482680 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 7bec8f4816b72fe84f3bf1daf6aade26 in 110ms, sequenceid=66, compaction requested=false; wal=null at 1733732482685 (+5 ms)Cleaning up temporary data from old regions at 1733732482687 (+2 ms)Region opened successfully at 1733732482691 (+4 ms) 2024-12-09T08:21:22,702 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7bec8f4816b72fe84f3bf1daf6aade26, disabling compactions & flushes 2024-12-09T08:21:22,702 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:22,702 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:22,702 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. after waiting 0 ms 2024-12-09T08:21:22,702 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:22,703 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732478095.7bec8f4816b72fe84f3bf1daf6aade26. 2024-12-09T08:21:22,703 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7bec8f4816b72fe84f3bf1daf6aade26: Waiting for close lock at 1733732482702Disabling compacts and flushes for region at 1733732482702Disabling writes for close at 1733732482702Writing region close event to WAL at 1733732482703 (+1 ms)Closed at 1733732482703 2024-12-09T08:21:22,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741934_1114 (size=93) 2024-12-09T08:21:22,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741934_1114 (size=93) 2024-12-09T08:21:22,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741934_1114 (size=93) 2024-12-09T08:21:22,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:21:22,708 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732482534) 2024-12-09T08:21:22,720 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=449 (was 441) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40523 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-767185396_22 at /127.0.0.1:37334 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-767185396_22 at /127.0.0.1:35618 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40523 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-767185396_22 at /127.0.0.1:37652 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 228), ProcessCount=11 (was 11), AvailableMemoryMB=3968 (was 4003) 2024-12-09T08:21:22,720 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-09T08:21:22,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T08:21:22,720 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:21:22,720 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:21:22,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:22,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:22,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:21:22,721 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T08:21:22,721 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=954470236, stopped=false 2024-12-09T08:21:22,722 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5bda946059d1,40865,1733732443726 2024-12-09T08:21:22,795 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:21:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:21:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:21:22,795 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:21:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:21:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:21:22,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T08:21:22,797 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:21:22,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:21:22,797 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:21:22,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:21:22,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:22,798 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,41239,1733732444531' ***** 2024-12-09T08:21:22,798 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T08:21:22,798 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,33703,1733732444647' ***** 2024-12-09T08:21:22,798 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T08:21:22,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:21:22,799 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:21:22,799 INFO [RS:1;5bda946059d1:33703 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T08:21:22,799 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:21:22,799 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:21:22,799 INFO [RS:1;5bda946059d1:33703 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T08:21:22,799 INFO [RS:0;5bda946059d1:41239 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T08:21:22,799 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:21:22,799 INFO [RS:0;5bda946059d1:41239 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T08:21:22,800 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(959): stopping server 5bda946059d1,33703,1733732444647 2024-12-09T08:21:22,800 INFO [RS:1;5bda946059d1:33703 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:21:22,800 INFO [RS:1;5bda946059d1:33703 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;5bda946059d1:33703. 2024-12-09T08:21:22,800 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(3091): Received CLOSE for 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:22,800 DEBUG [RS:1;5bda946059d1:33703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:21:22,800 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(959): stopping server 5bda946059d1,41239,1733732444531 2024-12-09T08:21:22,800 INFO [RS:0;5bda946059d1:41239 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:21:22,800 DEBUG [RS:1;5bda946059d1:33703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:22,800 INFO [RS:0;5bda946059d1:41239 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5bda946059d1:41239. 2024-12-09T08:21:22,800 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 920bbb1c4e65110d65293a979c5a15b1, disabling compactions & flushes 2024-12-09T08:21:22,801 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:22,800 INFO [RS:1;5bda946059d1:33703 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:21:22,801 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:22,801 INFO [RS:1;5bda946059d1:33703 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:21:22,801 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. after waiting 0 ms 2024-12-09T08:21:22,801 INFO [RS:1;5bda946059d1:33703 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:21:22,801 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:22,801 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T08:21:22,801 DEBUG [RS:0;5bda946059d1:41239 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:21:22,802 DEBUG [RS:0;5bda946059d1:41239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:22,802 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:21:22,802 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T08:21:22,802 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:21:22,802 DEBUG [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:21:22,802 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1325): Online Regions={920bbb1c4e65110d65293a979c5a15b1=testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1.} 2024-12-09T08:21:22,802 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T08:21:22,802 DEBUG [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1351): Waiting on 920bbb1c4e65110d65293a979c5a15b1 2024-12-09T08:21:22,802 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T08:21:22,802 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T08:21:22,802 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T08:21:22,802 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T08:21:22,803 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T08:21:22,811 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/default/testReplayEditsAfterRegionMovedWithMultiCF/920bbb1c4e65110d65293a979c5a15b1/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T08:21:22,812 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:22,812 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 920bbb1c4e65110d65293a979c5a15b1: Waiting for close lock at 1733732482800Running coprocessor pre-close hooks at 1733732482800Disabling compacts and flushes for region at 1733732482800Disabling writes for close at 1733732482801 (+1 ms)Writing region close event to WAL at 1733732482803 (+2 ms)Running coprocessor post-close hooks at 1733732482812 (+9 ms)Closed at 1733732482812 2024-12-09T08:21:22,812 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1. 2024-12-09T08:21:22,823 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:21:22,823 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:21:22,832 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/info/6b9fdd29933445aa912f1a85f3918cbc is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733732459510.920bbb1c4e65110d65293a979c5a15b1./info:regioninfo/1733732463066/Put/seqid=0 2024-12-09T08:21:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741938_1118 (size=8243) 2024-12-09T08:21:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741938_1118 (size=8243) 2024-12-09T08:21:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741938_1118 (size=8243) 2024-12-09T08:21:22,838 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/info/6b9fdd29933445aa912f1a85f3918cbc 2024-12-09T08:21:22,856 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/ns/724aa28212314aabadb76135e0b229d1 is 43, key is default/ns:d/1733732446967/Put/seqid=0 2024-12-09T08:21:22,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741939_1119 (size=5153) 2024-12-09T08:21:22,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741939_1119 (size=5153) 2024-12-09T08:21:22,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741939_1119 (size=5153) 2024-12-09T08:21:22,862 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/ns/724aa28212314aabadb76135e0b229d1 2024-12-09T08:21:22,878 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/table/7784ccea13214ae09654f3224795f283 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733732459936/Put/seqid=0 2024-12-09T08:21:22,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741940_1120 (size=5431) 2024-12-09T08:21:22,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741940_1120 (size=5431) 2024-12-09T08:21:22,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741940_1120 (size=5431) 2024-12-09T08:21:22,884 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/table/7784ccea13214ae09654f3224795f283 2024-12-09T08:21:22,890 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/info/6b9fdd29933445aa912f1a85f3918cbc as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/info/6b9fdd29933445aa912f1a85f3918cbc 2024-12-09T08:21:22,894 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/info/6b9fdd29933445aa912f1a85f3918cbc, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T08:21:22,895 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/ns/724aa28212314aabadb76135e0b229d1 as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/ns/724aa28212314aabadb76135e0b229d1 2024-12-09T08:21:22,900 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/ns/724aa28212314aabadb76135e0b229d1, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T08:21:22,901 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/.tmp/table/7784ccea13214ae09654f3224795f283 as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/table/7784ccea13214ae09654f3224795f283 2024-12-09T08:21:22,905 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/table/7784ccea13214ae09654f3224795f283, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T08:21:22,907 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=21, compaction requested=false 2024-12-09T08:21:22,911 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T08:21:22,912 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T08:21:22,912 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T08:21:22,912 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733732482802Running coprocessor pre-close hooks at 1733732482802Disabling compacts and flushes for region at 1733732482802Disabling writes for close at 1733732482802Obtaining lock to block concurrent updates at 1733732482803 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733732482803Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733732482804 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733732482809 (+5 ms)Flushing 1588230740/info: creating writer at 1733732482809Flushing 1588230740/info: appending metadata at 1733732482831 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733732482831Flushing 1588230740/ns: creating writer at 1733732482843 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733732482855 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733732482855Flushing 1588230740/table: creating writer at 1733732482866 (+11 ms)Flushing 1588230740/table: appending metadata at 1733732482877 (+11 ms)Flushing 1588230740/table: closing flushed file at 1733732482877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f762ec3: reopening flushed file at 1733732482889 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2830479: reopening flushed file at 1733732482895 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a54c49b: reopening flushed file at 1733732482900 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=21, compaction requested=false at 1733732482907 (+7 ms)Writing region close event to WAL at 1733732482908 (+1 ms)Running coprocessor post-close hooks at 1733732482911 (+3 ms)Closed at 1733732482912 (+1 ms) 2024-12-09T08:21:22,912 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T08:21:23,002 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,33703,1733732444647; all regions closed. 2024-12-09T08:21:23,002 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,41239,1733732444531; all regions closed. 2024-12-09T08:21:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741833_1009 (size=2652) 2024-12-09T08:21:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741833_1009 (size=2652) 2024-12-09T08:21:23,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741836_1012 (size=3783) 2024-12-09T08:21:23,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741836_1012 (size=3783) 2024-12-09T08:21:23,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741833_1009 (size=2652) 2024-12-09T08:21:23,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741836_1012 (size=3783) 2024-12-09T08:21:23,011 DEBUG [RS:0;5bda946059d1:41239 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C41239%2C1733732444531:(num 1733732446345) 2024-12-09T08:21:23,011 DEBUG [RS:0;5bda946059d1:41239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:21:23,011 DEBUG [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs 2024-12-09T08:21:23,011 INFO [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C33703%2C1733732444647.meta:.meta(num 1733732446763) 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:21:23,011 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:21:23,011 INFO [RS:0;5bda946059d1:41239 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:21:23,012 INFO [RS:0;5bda946059d1:41239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41239 2024-12-09T08:21:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741835_1011 (size=93) 2024-12-09T08:21:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741835_1011 (size=93) 2024-12-09T08:21:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741835_1011 (size=93) 2024-12-09T08:21:23,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:21:23,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,41239,1733732444531 2024-12-09T08:21:23,016 INFO [RS:0;5bda946059d1:41239 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:21:23,016 DEBUG [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/oldWALs 2024-12-09T08:21:23,016 INFO [RS:1;5bda946059d1:33703 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C33703%2C1733732444647:(num 1733732446346) 2024-12-09T08:21:23,016 DEBUG [RS:1;5bda946059d1:33703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:21:23,016 INFO [RS:1;5bda946059d1:33703 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:21:23,016 INFO [RS:1;5bda946059d1:33703 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:21:23,017 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,41239,1733732444531] 2024-12-09T08:21:23,017 INFO [RS:1;5bda946059d1:33703 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T08:21:23,017 INFO [RS:1;5bda946059d1:33703 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:21:23,017 INFO [RS:1;5bda946059d1:33703 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33703 2024-12-09T08:21:23,017 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:21:23,037 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,41239,1733732444531 already deleted, retry=false 2024-12-09T08:21:23,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:21:23,037 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,33703,1733732444647 2024-12-09T08:21:23,037 INFO [RS:1;5bda946059d1:33703 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:21:23,037 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5bda946059d1,41239,1733732444531 expired; onlineServers=1 2024-12-09T08:21:23,047 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,33703,1733732444647] 2024-12-09T08:21:23,058 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,33703,1733732444647 already deleted, retry=false 2024-12-09T08:21:23,058 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5bda946059d1,33703,1733732444647 expired; onlineServers=0 2024-12-09T08:21:23,058 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5bda946059d1,40865,1733732443726' ***** 2024-12-09T08:21:23,058 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T08:21:23,058 INFO [M:0;5bda946059d1:40865 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:21:23,058 INFO [M:0;5bda946059d1:40865 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:21:23,059 DEBUG [M:0;5bda946059d1:40865 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T08:21:23,059 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T08:21:23,059 DEBUG [M:0;5bda946059d1:40865 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T08:21:23,059 DEBUG [master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732445939 {}] cleaner.HFileCleaner(306): Exit Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732445939,5,FailOnTimeoutGroup] 2024-12-09T08:21:23,059 DEBUG [master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732445938 {}] cleaner.HFileCleaner(306): Exit Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732445938,5,FailOnTimeoutGroup] 2024-12-09T08:21:23,060 INFO [M:0;5bda946059d1:40865 {}] hbase.ChoreService(370): Chore service for: master/5bda946059d1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T08:21:23,060 INFO [M:0;5bda946059d1:40865 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:21:23,060 DEBUG [M:0;5bda946059d1:40865 {}] master.HMaster(1795): Stopping service threads 2024-12-09T08:21:23,060 INFO [M:0;5bda946059d1:40865 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T08:21:23,060 INFO [M:0;5bda946059d1:40865 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T08:21:23,061 INFO [M:0;5bda946059d1:40865 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T08:21:23,062 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T08:21:23,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T08:21:23,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:21:23,069 DEBUG [M:0;5bda946059d1:40865 {}] zookeeper.ZKUtil(347): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T08:21:23,069 WARN [M:0;5bda946059d1:40865 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T08:21:23,070 INFO [M:0;5bda946059d1:40865 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/.lastflushedseqids 2024-12-09T08:21:23,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741941_1121 (size=138) 2024-12-09T08:21:23,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741941_1121 (size=138) 2024-12-09T08:21:23,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741941_1121 (size=138) 2024-12-09T08:21:23,084 INFO [M:0;5bda946059d1:40865 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T08:21:23,085 INFO [M:0;5bda946059d1:40865 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T08:21:23,085 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T08:21:23,085 INFO [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:21:23,085 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:21:23,085 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T08:21:23,085 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:21:23,085 INFO [M:0;5bda946059d1:40865 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-09T08:21:23,103 DEBUG [M:0;5bda946059d1:40865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/352ec6fe6002417cb18e9e93a04b2744 is 82, key is hbase:meta,,1/info:regioninfo/1733732446851/Put/seqid=0 2024-12-09T08:21:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741942_1122 (size=5672) 2024-12-09T08:21:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741942_1122 (size=5672) 2024-12-09T08:21:23,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741942_1122 (size=5672) 2024-12-09T08:21:23,112 INFO [M:0;5bda946059d1:40865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/352ec6fe6002417cb18e9e93a04b2744 2024-12-09T08:21:23,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:23,126 INFO [RS:0;5bda946059d1:41239 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:21:23,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41239-0x10009a9aa0d0001, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:23,126 INFO [RS:0;5bda946059d1:41239 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,41239,1733732444531; zookeeper connection closed. 2024-12-09T08:21:23,127 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d0fa16d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d0fa16d 2024-12-09T08:21:23,128 DEBUG [M:0;5bda946059d1:40865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eda1135003384b4898b08c6dff98aa9f is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733732459941/Put/seqid=0 2024-12-09T08:21:23,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741943_1123 (size=7756) 2024-12-09T08:21:23,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741943_1123 (size=7756) 2024-12-09T08:21:23,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741943_1123 (size=7756) 2024-12-09T08:21:23,135 INFO [M:0;5bda946059d1:40865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eda1135003384b4898b08c6dff98aa9f 2024-12-09T08:21:23,138 INFO [M:0;5bda946059d1:40865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eda1135003384b4898b08c6dff98aa9f 2024-12-09T08:21:23,148 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:23,148 INFO [RS:1;5bda946059d1:33703 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:21:23,148 INFO [RS:1;5bda946059d1:33703 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,33703,1733732444647; zookeeper connection closed. 2024-12-09T08:21:23,148 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33703-0x10009a9aa0d0002, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:23,148 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66020394 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66020394 2024-12-09T08:21:23,148 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T08:21:23,151 DEBUG [M:0;5bda946059d1:40865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/961c214ebe3c441a84f52927c971fd4c is 69, key is 5bda946059d1,33703,1733732444647/rs:state/1733732446029/Put/seqid=0 2024-12-09T08:21:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741944_1124 (size=5440) 2024-12-09T08:21:23,157 INFO [M:0;5bda946059d1:40865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/961c214ebe3c441a84f52927c971fd4c 2024-12-09T08:21:23,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741944_1124 (size=5440) 2024-12-09T08:21:23,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741944_1124 (size=5440) 2024-12-09T08:21:23,161 INFO [M:0;5bda946059d1:40865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 961c214ebe3c441a84f52927c971fd4c 2024-12-09T08:21:23,162 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/352ec6fe6002417cb18e9e93a04b2744 as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/352ec6fe6002417cb18e9e93a04b2744 2024-12-09T08:21:23,167 INFO [M:0;5bda946059d1:40865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/352ec6fe6002417cb18e9e93a04b2744, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T08:21:23,168 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eda1135003384b4898b08c6dff98aa9f as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eda1135003384b4898b08c6dff98aa9f 2024-12-09T08:21:23,173 INFO [M:0;5bda946059d1:40865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eda1135003384b4898b08c6dff98aa9f 2024-12-09T08:21:23,173 INFO [M:0;5bda946059d1:40865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eda1135003384b4898b08c6dff98aa9f, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T08:21:23,174 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/961c214ebe3c441a84f52927c971fd4c as hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/961c214ebe3c441a84f52927c971fd4c 2024-12-09T08:21:23,178 INFO [M:0;5bda946059d1:40865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 961c214ebe3c441a84f52927c971fd4c 2024-12-09T08:21:23,178 INFO [M:0;5bda946059d1:40865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40523/user/jenkins/test-data/4d072661-9d8b-5bbd-a316-d085f106c609/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/961c214ebe3c441a84f52927c971fd4c, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T08:21:23,179 INFO [M:0;5bda946059d1:40865 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 94ms, sequenceid=168, compaction requested=false 2024-12-09T08:21:23,180 INFO [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:21:23,180 DEBUG [M:0;5bda946059d1:40865 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733732483085Disabling compacts and flushes for region at 1733732483085Disabling writes for close at 1733732483085Obtaining lock to block concurrent updates at 1733732483085Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733732483085Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733732483086 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733732483087 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733732483087Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733732483103 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733732483103Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733732483116 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733732483128 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733732483128Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733732483138 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733732483151 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733732483151Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2efbb48e: reopening flushed file at 1733732483161 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47aa1c30: reopening flushed file at 1733732483167 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e42501a: reopening flushed file at 1733732483173 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 94ms, sequenceid=168, compaction requested=false at 1733732483179 (+6 ms)Writing region close event to WAL at 1733732483180 (+1 ms)Closed at 1733732483180 2024-12-09T08:21:23,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741830_1006 (size=71349) 2024-12-09T08:21:23,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41795 is added to blk_1073741830_1006 (size=71349) 2024-12-09T08:21:23,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38773 is added to blk_1073741830_1006 (size=71349) 2024-12-09T08:21:23,184 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:21:23,184 INFO [M:0;5bda946059d1:40865 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T08:21:23,184 INFO [M:0;5bda946059d1:40865 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40865 2024-12-09T08:21:23,184 INFO [M:0;5bda946059d1:40865 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:21:23,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:23,354 INFO [M:0;5bda946059d1:40865 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:21:23,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40865-0x10009a9aa0d0000, quorum=127.0.0.1:61985, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:21:23,361 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463597 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463597 (inode 16655) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732463354/wal.1733732463597 (inode 16655) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:21:23,363 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732455098/wal.1733732455235 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:23,363 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732478094/wal.1733732478331 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:23,366 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732472986 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732472986 (inode 16678) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732463734/wal.1733732472986 (inode 16678) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:21:23,367 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732448012/wal.1733732448080 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:23,369 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473574 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473574 (inode 16704) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732473165/wal.1733732473574 (inode 16704) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:21:23,369 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732473701/wal.1733732473749 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:21:23,371 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal.1733732447891 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal.1733732447891 (inode 16485) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732447716/wal.1733732447891 (inode 16485) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:21:23,374 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal.1733732447483 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal.1733732447483 (inode 16462) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732447249/wal.1733732447483 (inode 16462) Holder DFSClient_NONMAPREDUCE_361199537_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:21:23,412 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:21:23,415 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:21:23,415 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:21:23,416 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:21:23,416 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,STOPPED} 2024-12-09T08:21:23,419 WARN [BP-361248767-172.17.0.2-1733732439363 heartbeating to localhost/127.0.0.1:40523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:21:23,419 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:21:23,419 WARN [BP-361248767-172.17.0.2-1733732439363 heartbeating to localhost/127.0.0.1:40523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-361248767-172.17.0.2-1733732439363 (Datanode Uuid 0a71e0a5-6836-4c09-b525-c241270ea2a3) service to localhost/127.0.0.1:40523 2024-12-09T08:21:23,419 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:21:23,420 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data5/current/BP-361248767-172.17.0.2-1733732439363 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:21:23,420 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data6/current/BP-361248767-172.17.0.2-1733732439363 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:21:23,421 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:21:23,422 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:21:23,422 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:21:23,423 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:21:23,423 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:21:23,423 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,STOPPED} 2024-12-09T08:21:23,424 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:21:23,424 WARN [BP-361248767-172.17.0.2-1733732439363 heartbeating to localhost/127.0.0.1:40523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:21:23,424 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:21:23,424 WARN [BP-361248767-172.17.0.2-1733732439363 heartbeating to localhost/127.0.0.1:40523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-361248767-172.17.0.2-1733732439363 (Datanode Uuid efcf3e6d-2d12-4da2-b536-d812b4be17aa) service to localhost/127.0.0.1:40523 2024-12-09T08:21:23,425 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data3/current/BP-361248767-172.17.0.2-1733732439363 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:21:23,425 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data4/current/BP-361248767-172.17.0.2-1733732439363 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:21:23,425 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:21:23,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:21:23,427 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:21:23,427 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:21:23,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:21:23,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,STOPPED} 2024-12-09T08:21:23,429 WARN [BP-361248767-172.17.0.2-1733732439363 heartbeating to localhost/127.0.0.1:40523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:21:23,429 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:21:23,429 WARN [BP-361248767-172.17.0.2-1733732439363 heartbeating to localhost/127.0.0.1:40523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-361248767-172.17.0.2-1733732439363 (Datanode Uuid 581cf5ca-7eed-440c-9140-bcdd1fc65c2c) service to localhost/127.0.0.1:40523 2024-12-09T08:21:23,429 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:21:23,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data1/current/BP-361248767-172.17.0.2-1733732439363 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:21:23,430 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/cluster_ad6e4c29-5e21-8c3a-3d57-546d544c1cff/data/data2/current/BP-361248767-172.17.0.2-1733732439363 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:21:23,430 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:21:23,436 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T08:21:23,437 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:21:23,437 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:21:23,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:21:23,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4bb38c1a-1696-3fcd-5b17-3c705d66911c/hadoop.log.dir/,STOPPED} 2024-12-09T08:21:23,448 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T08:21:23,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down