2024-12-10 06:28:07,241 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-10 06:28:07,257 main DEBUG Took 0.013324 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 06:28:07,258 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 06:28:07,258 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 06:28:07,259 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 06:28:07,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,268 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 06:28:07,283 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,285 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,287 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,290 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,291 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,292 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,293 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,294 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,295 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,296 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,297 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 06:28:07,298 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,299 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 06:28:07,301 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 06:28:07,302 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 06:28:07,304 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 06:28:07,305 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 06:28:07,306 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 06:28:07,307 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 06:28:07,316 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 06:28:07,319 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 06:28:07,321 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 06:28:07,321 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 06:28:07,322 main DEBUG createAppenders(={Console}) 2024-12-10 06:28:07,322 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-10 06:28:07,323 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-10 06:28:07,323 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-10 06:28:07,324 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 06:28:07,324 main DEBUG OutputStream closed 2024-12-10 06:28:07,325 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 06:28:07,325 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 06:28:07,325 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-10 06:28:07,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 06:28:07,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 06:28:07,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 06:28:07,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 06:28:07,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 06:28:07,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 06:28:07,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 06:28:07,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 06:28:07,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 06:28:07,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 06:28:07,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 06:28:07,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 06:28:07,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 06:28:07,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 06:28:07,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 06:28:07,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 06:28:07,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 06:28:07,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 06:28:07,426 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 06:28:07,427 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-10 06:28:07,427 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 06:28:07,428 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-10T06:28:07,675 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54 2024-12-10 06:28:07,678 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 06:28:07,678 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T06:28:07,687 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-10T06:28:07,693 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-10T06:28:07,717 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T06:28:07,779 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T06:28:07,779 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T06:28:07,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T06:28:07,809 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294, deleteOnExit=true 2024-12-10T06:28:07,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T06:28:07,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/test.cache.data in system properties and HBase conf 2024-12-10T06:28:07,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T06:28:07,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir in system properties and HBase conf 2024-12-10T06:28:07,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T06:28:07,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T06:28:07,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T06:28:07,911 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T06:28:08,052 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T06:28:08,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T06:28:08,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T06:28:08,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T06:28:08,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T06:28:08,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T06:28:08,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T06:28:08,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T06:28:08,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T06:28:08,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T06:28:08,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/nfs.dump.dir in system properties and HBase conf 2024-12-10T06:28:08,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/java.io.tmpdir in system properties and HBase conf 2024-12-10T06:28:08,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T06:28:08,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T06:28:08,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T06:28:09,117 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T06:28:09,192 INFO [Time-limited test {}] log.Log(170): Logging initialized @2800ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T06:28:09,277 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:28:09,340 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T06:28:09,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T06:28:09,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T06:28:09,366 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T06:28:09,383 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:28:09,386 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,AVAILABLE} 2024-12-10T06:28:09,387 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T06:28:09,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/java.io.tmpdir/jetty-localhost-38375-hadoop-hdfs-3_4_1-tests_jar-_-any-474824815034614473/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T06:28:09,596 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:38375} 2024-12-10T06:28:09,597 INFO [Time-limited test {}] server.Server(415): Started @3205ms 2024-12-10T06:28:10,105 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:28:10,112 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T06:28:10,113 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T06:28:10,113 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T06:28:10,114 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T06:28:10,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,AVAILABLE} 2024-12-10T06:28:10,115 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T06:28:10,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/java.io.tmpdir/jetty-localhost-41059-hadoop-hdfs-3_4_1-tests_jar-_-any-17456490254899027683/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:28:10,235 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:41059} 2024-12-10T06:28:10,235 INFO [Time-limited test {}] server.Server(415): Started @3843ms 2024-12-10T06:28:10,290 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T06:28:10,407 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:28:10,414 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T06:28:10,421 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T06:28:10,421 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T06:28:10,421 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T06:28:10,422 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,AVAILABLE} 2024-12-10T06:28:10,423 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T06:28:10,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/java.io.tmpdir/jetty-localhost-36499-hadoop-hdfs-3_4_1-tests_jar-_-any-7662986558880139853/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:28:10,545 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:36499} 2024-12-10T06:28:10,545 INFO [Time-limited test {}] server.Server(415): Started @4153ms 2024-12-10T06:28:10,548 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T06:28:10,596 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T06:28:10,601 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T06:28:10,603 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T06:28:10,604 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T06:28:10,604 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T06:28:10,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,AVAILABLE} 2024-12-10T06:28:10,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T06:28:10,731 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/java.io.tmpdir/jetty-localhost-35425-hadoop-hdfs-3_4_1-tests_jar-_-any-9875251933427977708/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:28:10,732 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:35425} 2024-12-10T06:28:10,732 INFO [Time-limited test {}] server.Server(415): Started @4341ms 2024-12-10T06:28:10,735 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T06:28:12,316 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data2/current/BP-616036786-172.17.0.2-1733812088637/current, will proceed with Du for space computation calculation, 2024-12-10T06:28:12,316 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data1/current/BP-616036786-172.17.0.2-1733812088637/current, will proceed with Du for space computation calculation, 2024-12-10T06:28:12,325 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data3/current/BP-616036786-172.17.0.2-1733812088637/current, will proceed with Du for space computation calculation, 2024-12-10T06:28:12,329 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data4/current/BP-616036786-172.17.0.2-1733812088637/current, will proceed with Du for space computation calculation, 2024-12-10T06:28:12,357 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T06:28:12,356 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T06:28:12,413 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f72f530d6780289 with lease ID 0xf7025dc2e42169c7: Processing first storage report for DS-3840f433-b6cc-44df-9edf-8679ca4e88a0 from datanode DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e03296b0-875f-446a-93af-aef12208d63e, infoPort=33175, infoSecurePort=0, ipcPort=34845, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637) 2024-12-10T06:28:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f72f530d6780289 with lease ID 0xf7025dc2e42169c7: from storage DS-3840f433-b6cc-44df-9edf-8679ca4e88a0 node DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e03296b0-875f-446a-93af-aef12208d63e, infoPort=33175, infoSecurePort=0, ipcPort=34845, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-10T06:28:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf05d95deecef6ca with lease ID 0xf7025dc2e42169c8: Processing first storage report for DS-12b57370-5005-4de0-ba73-054d878d9d64 from datanode DatanodeRegistration(127.0.0.1:45191, datanodeUuid=3c5c76b2-33e8-4b82-af04-f232a7b8eaa7, infoPort=33133, infoSecurePort=0, ipcPort=44593, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637) 2024-12-10T06:28:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf05d95deecef6ca with lease ID 0xf7025dc2e42169c8: from storage DS-12b57370-5005-4de0-ba73-054d878d9d64 node DatanodeRegistration(127.0.0.1:45191, datanodeUuid=3c5c76b2-33e8-4b82-af04-f232a7b8eaa7, infoPort=33133, infoSecurePort=0, ipcPort=44593, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T06:28:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f72f530d6780289 with lease ID 0xf7025dc2e42169c7: Processing first storage report for DS-4ad872d4-0069-469d-9294-3d9072403890 from datanode DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e03296b0-875f-446a-93af-aef12208d63e, infoPort=33175, infoSecurePort=0, ipcPort=34845, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637) 2024-12-10T06:28:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f72f530d6780289 with lease ID 0xf7025dc2e42169c7: from storage DS-4ad872d4-0069-469d-9294-3d9072403890 node DatanodeRegistration(127.0.0.1:40347, datanodeUuid=e03296b0-875f-446a-93af-aef12208d63e, infoPort=33175, infoSecurePort=0, ipcPort=34845, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T06:28:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf05d95deecef6ca with lease ID 0xf7025dc2e42169c8: Processing first storage report for DS-2e56cd9f-d452-4706-bb61-baa251a17b79 from datanode DatanodeRegistration(127.0.0.1:45191, datanodeUuid=3c5c76b2-33e8-4b82-af04-f232a7b8eaa7, infoPort=33133, infoSecurePort=0, ipcPort=44593, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637) 2024-12-10T06:28:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf05d95deecef6ca with lease ID 0xf7025dc2e42169c8: from storage DS-2e56cd9f-d452-4706-bb61-baa251a17b79 node DatanodeRegistration(127.0.0.1:45191, datanodeUuid=3c5c76b2-33e8-4b82-af04-f232a7b8eaa7, infoPort=33133, infoSecurePort=0, ipcPort=44593, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T06:28:12,485 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data5/current/BP-616036786-172.17.0.2-1733812088637/current, will proceed with Du for space computation calculation, 2024-12-10T06:28:12,485 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data6/current/BP-616036786-172.17.0.2-1733812088637/current, will proceed with Du for space computation calculation, 2024-12-10T06:28:12,504 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T06:28:12,508 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefd9f9fc24794dbb with lease ID 0xf7025dc2e42169c9: Processing first storage report for DS-f17827d8-43e9-4c4d-a644-e003bd771ad2 from datanode DatanodeRegistration(127.0.0.1:33683, datanodeUuid=ebf1f6bc-fd40-4f24-9c9d-8bf323c1bc33, infoPort=40515, infoSecurePort=0, ipcPort=40571, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637) 2024-12-10T06:28:12,508 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefd9f9fc24794dbb with lease ID 0xf7025dc2e42169c9: from storage DS-f17827d8-43e9-4c4d-a644-e003bd771ad2 node DatanodeRegistration(127.0.0.1:33683, datanodeUuid=ebf1f6bc-fd40-4f24-9c9d-8bf323c1bc33, infoPort=40515, infoSecurePort=0, ipcPort=40571, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T06:28:12,509 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefd9f9fc24794dbb with lease ID 0xf7025dc2e42169c9: Processing first storage report for DS-1018ff46-2d12-4c53-b5c7-a695afe01674 from datanode DatanodeRegistration(127.0.0.1:33683, datanodeUuid=ebf1f6bc-fd40-4f24-9c9d-8bf323c1bc33, infoPort=40515, infoSecurePort=0, ipcPort=40571, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637) 2024-12-10T06:28:12,509 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefd9f9fc24794dbb with lease ID 0xf7025dc2e42169c9: from storage DS-1018ff46-2d12-4c53-b5c7-a695afe01674 node DatanodeRegistration(127.0.0.1:33683, datanodeUuid=ebf1f6bc-fd40-4f24-9c9d-8bf323c1bc33, infoPort=40515, infoSecurePort=0, ipcPort=40571, storageInfo=lv=-57;cid=testClusterID;nsid=837529263;c=1733812088637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T06:28:12,606 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54 2024-12-10T06:28:12,693 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/zookeeper_0, clientPort=59695, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T06:28:12,703 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59695 2024-12-10T06:28:12,713 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:12,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:12,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741825_1001 (size=7) 2024-12-10T06:28:12,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741825_1001 (size=7) 2024-12-10T06:28:12,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741825_1001 (size=7) 2024-12-10T06:28:13,362 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25 with version=8 2024-12-10T06:28:13,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/hbase-staging 2024-12-10T06:28:13,676 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c42c682b34f7:0 server-side Connection retries=45 2024-12-10T06:28:13,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:13,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:13,692 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T06:28:13,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:13,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T06:28:13,831 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T06:28:13,892 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T06:28:13,900 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T06:28:13,904 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T06:28:13,932 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4111 (auto-detected) 2024-12-10T06:28:13,933 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T06:28:13,953 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34871 2024-12-10T06:28:13,973 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34871 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-10T06:28:14,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:348710x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T06:28:14,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34871-0x1019b0ee98c0000 connected 2024-12-10T06:28:14,282 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,296 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:14,300 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25, hbase.cluster.distributed=false 2024-12-10T06:28:14,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T06:28:14,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34871 2024-12-10T06:28:14,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34871 2024-12-10T06:28:14,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34871 2024-12-10T06:28:14,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34871 2024-12-10T06:28:14,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34871 2024-12-10T06:28:14,443 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c42c682b34f7:0 server-side Connection retries=45 2024-12-10T06:28:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,445 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T06:28:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,445 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T06:28:14,448 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T06:28:14,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T06:28:14,451 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45385 2024-12-10T06:28:14,453 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45385 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-10T06:28:14,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,457 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453850x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T06:28:14,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45385-0x1019b0ee98c0001 connected 2024-12-10T06:28:14,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:14,478 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T06:28:14,485 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T06:28:14,487 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T06:28:14,492 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T06:28:14,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45385 2024-12-10T06:28:14,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45385 2024-12-10T06:28:14,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45385 2024-12-10T06:28:14,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45385 2024-12-10T06:28:14,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45385 2024-12-10T06:28:14,512 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c42c682b34f7:0 server-side Connection retries=45 2024-12-10T06:28:14,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,512 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T06:28:14,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T06:28:14,513 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T06:28:14,513 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T06:28:14,514 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41301 2024-12-10T06:28:14,516 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41301 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-10T06:28:14,517 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,520 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413010x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T06:28:14,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413010x0, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:14,538 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41301-0x1019b0ee98c0002 connected 2024-12-10T06:28:14,538 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T06:28:14,539 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T06:28:14,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T06:28:14,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T06:28:14,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41301 2024-12-10T06:28:14,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41301 2024-12-10T06:28:14,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41301 2024-12-10T06:28:14,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41301 2024-12-10T06:28:14,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41301 2024-12-10T06:28:14,566 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c42c682b34f7:0 server-side Connection retries=45 2024-12-10T06:28:14,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,566 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,566 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T06:28:14,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T06:28:14,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T06:28:14,567 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T06:28:14,567 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T06:28:14,568 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35565 2024-12-10T06:28:14,570 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35565 connecting to ZooKeeper ensemble=127.0.0.1:59695 2024-12-10T06:28:14,571 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,573 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355650x0, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T06:28:14,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355650x0, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:14,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35565-0x1019b0ee98c0003 connected 2024-12-10T06:28:14,614 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T06:28:14,615 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T06:28:14,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T06:28:14,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T06:28:14,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35565 2024-12-10T06:28:14,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35565 2024-12-10T06:28:14,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35565 2024-12-10T06:28:14,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35565 2024-12-10T06:28:14,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35565 2024-12-10T06:28:14,636 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c42c682b34f7:34871 2024-12-10T06:28:14,637 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c42c682b34f7,34871,1733812093474 2024-12-10T06:28:14,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,660 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c42c682b34f7,34871,1733812093474 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,697 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T06:28:14,699 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c42c682b34f7,34871,1733812093474 from backup master directory 2024-12-10T06:28:14,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c42c682b34f7,34871,1733812093474 2024-12-10T06:28:14,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T06:28:14,712 WARN [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T06:28:14,713 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c42c682b34f7,34871,1733812093474 2024-12-10T06:28:14,715 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T06:28:14,716 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T06:28:14,774 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/hbase.id] with ID: 2c533b3b-6bbc-4d1b-8169-651bba667a42 2024-12-10T06:28:14,775 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/.tmp/hbase.id 2024-12-10T06:28:14,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741826_1002 (size=42) 2024-12-10T06:28:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741826_1002 (size=42) 2024-12-10T06:28:14,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741826_1002 (size=42) 2024-12-10T06:28:14,792 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/.tmp/hbase.id]:[hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/hbase.id] 2024-12-10T06:28:14,833 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T06:28:14,838 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T06:28:14,857 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-10T06:28:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:14,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741827_1003 (size=196) 2024-12-10T06:28:14,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741827_1003 (size=196) 2024-12-10T06:28:14,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741827_1003 (size=196) 2024-12-10T06:28:14,910 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:28:14,912 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T06:28:14,918 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:14,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741828_1004 (size=1189) 2024-12-10T06:28:14,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741828_1004 (size=1189) 2024-12-10T06:28:14,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741828_1004 (size=1189) 2024-12-10T06:28:14,972 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store 2024-12-10T06:28:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741829_1005 (size=34) 2024-12-10T06:28:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741829_1005 (size=34) 2024-12-10T06:28:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741829_1005 (size=34) 2024-12-10T06:28:15,002 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T06:28:15,005 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:15,006 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T06:28:15,006 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:15,006 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:15,008 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T06:28:15,008 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:15,008 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:15,009 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733812095006Disabling compacts and flushes for region at 1733812095006Disabling writes for close at 1733812095008 (+2 ms)Writing region close event to WAL at 1733812095008Closed at 1733812095008 2024-12-10T06:28:15,011 WARN [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/.initializing 2024-12-10T06:28:15,011 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/WALs/c42c682b34f7,34871,1733812093474 2024-12-10T06:28:15,019 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:15,035 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C34871%2C1733812093474, suffix=, logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/WALs/c42c682b34f7,34871,1733812093474, archiveDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/oldWALs, maxLogs=10 2024-12-10T06:28:15,066 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/WALs/c42c682b34f7,34871,1733812093474/c42c682b34f7%2C34871%2C1733812093474.1733812095039, exclude list is [], retry=0 2024-12-10T06:28:15,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:15,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:15,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:15,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:15,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T06:28:15,132 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/WALs/c42c682b34f7,34871,1733812093474/c42c682b34f7%2C34871%2C1733812093474.1733812095039 2024-12-10T06:28:15,133 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:15,133 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:15,134 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:15,137 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,138 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T06:28:15,204 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:15,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:15,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T06:28:15,210 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:15,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:15,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T06:28:15,214 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:15,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:15,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T06:28:15,219 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:15,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:15,220 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,223 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,224 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,229 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,230 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,233 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T06:28:15,236 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T06:28:15,240 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:15,241 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74602546, jitterRate=0.11166456341743469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T06:28:15,247 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733812095150Initializing all the Stores at 1733812095152 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812095153 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812095153Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812095154 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812095154Cleaning up temporary data from old regions at 1733812095230 (+76 ms)Region opened successfully at 1733812095247 (+17 ms) 2024-12-10T06:28:15,248 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T06:28:15,282 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1936f66a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c42c682b34f7/172.17.0.2:0 2024-12-10T06:28:15,315 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T06:28:15,326 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T06:28:15,326 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T06:28:15,330 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T06:28:15,331 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T06:28:15,336 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-10T06:28:15,336 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T06:28:15,362 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T06:28:15,370 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T06:28:15,380 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T06:28:15,383 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T06:28:15,385 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T06:28:15,395 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T06:28:15,398 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T06:28:15,401 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T06:28:15,411 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T06:28:15,412 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T06:28:15,427 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T06:28:15,446 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T06:28:15,457 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,476 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c42c682b34f7,34871,1733812093474, sessionid=0x1019b0ee98c0000, setting cluster-up flag (Was=false) 2024-12-10T06:28:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,557 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T06:28:15,559 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c42c682b34f7,34871,1733812093474 2024-12-10T06:28:15,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:15,643 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T06:28:15,644 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c42c682b34f7,34871,1733812093474 2024-12-10T06:28:15,651 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T06:28:15,724 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(746): ClusterId : 2c533b3b-6bbc-4d1b-8169-651bba667a42 2024-12-10T06:28:15,724 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(746): ClusterId : 2c533b3b-6bbc-4d1b-8169-651bba667a42 2024-12-10T06:28:15,724 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(746): ClusterId : 2c533b3b-6bbc-4d1b-8169-651bba667a42 2024-12-10T06:28:15,727 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T06:28:15,727 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T06:28:15,727 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T06:28:15,727 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T06:28:15,737 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T06:28:15,745 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T06:28:15,751 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c42c682b34f7,34871,1733812093474 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T06:28:15,760 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T06:28:15,761 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T06:28:15,761 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T06:28:15,761 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T06:28:15,761 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T06:28:15,761 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T06:28:15,761 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c42c682b34f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c42c682b34f7:0, corePoolSize=10, maxPoolSize=10 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c42c682b34f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T06:28:15,762 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:15,768 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733812125768 2024-12-10T06:28:15,768 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T06:28:15,768 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T06:28:15,770 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T06:28:15,771 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T06:28:15,774 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:15,774 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T06:28:15,775 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T06:28:15,776 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T06:28:15,776 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T06:28:15,777 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T06:28:15,777 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:15,781 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T06:28:15,782 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T06:28:15,783 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T06:28:15,817 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T06:28:15,817 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T06:28:15,817 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T06:28:15,817 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T06:28:15,818 DEBUG [RS:1;c42c682b34f7:41301 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273487de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c42c682b34f7/172.17.0.2:0 2024-12-10T06:28:15,818 DEBUG [RS:2;c42c682b34f7:35565 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@782b93cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c42c682b34f7/172.17.0.2:0 2024-12-10T06:28:15,818 DEBUG [RS:0;c42c682b34f7:45385 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d396408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c42c682b34f7/172.17.0.2:0 2024-12-10T06:28:15,818 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T06:28:15,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741831_1007 (size=1321) 2024-12-10T06:28:15,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741831_1007 (size=1321) 2024-12-10T06:28:15,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741831_1007 (size=1321) 2024-12-10T06:28:15,824 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.large.0-1733812095820,5,FailOnTimeoutGroup] 2024-12-10T06:28:15,829 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.small.0-1733812095825,5,FailOnTimeoutGroup] 2024-12-10T06:28:15,829 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:15,829 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T06:28:15,830 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:15,831 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:15,832 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T06:28:15,833 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25 2024-12-10T06:28:15,837 DEBUG [RS:2;c42c682b34f7:35565 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c42c682b34f7:35565 2024-12-10T06:28:15,838 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c42c682b34f7:41301 2024-12-10T06:28:15,840 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c42c682b34f7:45385 2024-12-10T06:28:15,840 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T06:28:15,840 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T06:28:15,840 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T06:28:15,840 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T06:28:15,841 DEBUG [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T06:28:15,841 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T06:28:15,841 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T06:28:15,841 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T06:28:15,842 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T06:28:15,844 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(2659): reportForDuty to master=c42c682b34f7,34871,1733812093474 with port=41301, startcode=1733812094511 2024-12-10T06:28:15,844 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(2659): reportForDuty to master=c42c682b34f7,34871,1733812093474 with port=35565, startcode=1733812094566 2024-12-10T06:28:15,844 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(2659): reportForDuty to master=c42c682b34f7,34871,1733812093474 with port=45385, startcode=1733812094403 2024-12-10T06:28:15,858 DEBUG [RS:0;c42c682b34f7:45385 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T06:28:15,858 DEBUG [RS:2;c42c682b34f7:35565 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T06:28:15,858 DEBUG [RS:1;c42c682b34f7:41301 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T06:28:15,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741832_1008 (size=32) 2024-12-10T06:28:15,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741832_1008 (size=32) 2024-12-10T06:28:15,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741832_1008 (size=32) 2024-12-10T06:28:15,893 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55933, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T06:28:15,893 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54399, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T06:28:15,893 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T06:28:15,899 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c42c682b34f7,45385,1733812094403 2024-12-10T06:28:15,901 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34871 {}] master.ServerManager(517): Registering regionserver=c42c682b34f7,45385,1733812094403 2024-12-10T06:28:15,913 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c42c682b34f7,35565,1733812094566 2024-12-10T06:28:15,913 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34871 {}] master.ServerManager(517): Registering regionserver=c42c682b34f7,35565,1733812094566 2024-12-10T06:28:15,916 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25 2024-12-10T06:28:15,916 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40333 2024-12-10T06:28:15,917 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T06:28:15,918 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34871 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c42c682b34f7,41301,1733812094511 2024-12-10T06:28:15,918 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34871 {}] master.ServerManager(517): Registering regionserver=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:15,918 DEBUG [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25 2024-12-10T06:28:15,918 DEBUG [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40333 2024-12-10T06:28:15,918 DEBUG [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T06:28:15,921 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25 2024-12-10T06:28:15,921 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40333 2024-12-10T06:28:15,921 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T06:28:15,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T06:28:15,957 DEBUG [RS:0;c42c682b34f7:45385 {}] zookeeper.ZKUtil(111): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c42c682b34f7,45385,1733812094403 2024-12-10T06:28:15,957 DEBUG [RS:2;c42c682b34f7:35565 {}] zookeeper.ZKUtil(111): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c42c682b34f7,35565,1733812094566 2024-12-10T06:28:15,958 WARN [RS:0;c42c682b34f7:45385 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T06:28:15,958 WARN [RS:2;c42c682b34f7:35565 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T06:28:15,958 INFO [RS:0;c42c682b34f7:45385 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:15,958 DEBUG [RS:1;c42c682b34f7:41301 {}] zookeeper.ZKUtil(111): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c42c682b34f7,41301,1733812094511 2024-12-10T06:28:15,958 INFO [RS:2;c42c682b34f7:35565 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:15,958 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403 2024-12-10T06:28:15,958 WARN [RS:1;c42c682b34f7:41301 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T06:28:15,958 DEBUG [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,35565,1733812094566 2024-12-10T06:28:15,958 INFO [RS:1;c42c682b34f7:41301 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:15,958 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511 2024-12-10T06:28:15,959 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c42c682b34f7,41301,1733812094511] 2024-12-10T06:28:15,960 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c42c682b34f7,35565,1733812094566] 2024-12-10T06:28:15,960 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c42c682b34f7,45385,1733812094403] 2024-12-10T06:28:15,986 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T06:28:15,986 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T06:28:15,986 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T06:28:15,999 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T06:28:15,999 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T06:28:16,000 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T06:28:16,005 INFO [RS:0;c42c682b34f7:45385 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T06:28:16,005 INFO [RS:2;c42c682b34f7:35565 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T06:28:16,005 INFO [RS:1;c42c682b34f7:41301 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T06:28:16,006 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,006 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,006 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,007 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T06:28:16,007 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T06:28:16,007 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T06:28:16,014 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T06:28:16,014 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T06:28:16,014 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T06:28:16,015 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,015 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,015 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,015 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,015 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,015 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,015 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T06:28:16,016 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T06:28:16,017 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,016 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T06:28:16,017 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:28:16,017 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:0;c42c682b34f7:45385 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:28:16,017 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,017 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,018 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,018 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,018 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c42c682b34f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T06:28:16,018 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:28:16,018 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:28:16,018 DEBUG [RS:1;c42c682b34f7:41301 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:28:16,018 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,018 DEBUG [RS:2;c42c682b34f7:35565 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c42c682b34f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T06:28:16,018 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,018 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,018 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,019 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,019 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,45385,1733812094403-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T06:28:16,022 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,022 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,022 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,022 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,022 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,022 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,41301,1733812094511-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T06:28:16,023 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,023 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,023 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,023 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,023 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,023 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,35565,1733812094566-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T06:28:16,044 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T06:28:16,046 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,45385,1733812094403-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,046 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,046 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.Replication(171): c42c682b34f7,45385,1733812094403 started 2024-12-10T06:28:16,047 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T06:28:16,047 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T06:28:16,047 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,35565,1733812094566-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,047 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,41301,1733812094511-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,047 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,047 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,048 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.Replication(171): c42c682b34f7,41301,1733812094511 started 2024-12-10T06:28:16,048 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.Replication(171): c42c682b34f7,35565,1733812094566 started 2024-12-10T06:28:16,078 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,078 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,078 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,078 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(1482): Serving as c42c682b34f7,35565,1733812094566, RpcServer on c42c682b34f7/172.17.0.2:35565, sessionid=0x1019b0ee98c0003 2024-12-10T06:28:16,078 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1482): Serving as c42c682b34f7,45385,1733812094403, RpcServer on c42c682b34f7/172.17.0.2:45385, sessionid=0x1019b0ee98c0001 2024-12-10T06:28:16,078 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1482): Serving as c42c682b34f7,41301,1733812094511, RpcServer on c42c682b34f7/172.17.0.2:41301, sessionid=0x1019b0ee98c0002 2024-12-10T06:28:16,079 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T06:28:16,079 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T06:28:16,079 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T06:28:16,079 DEBUG [RS:1;c42c682b34f7:41301 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c42c682b34f7,41301,1733812094511 2024-12-10T06:28:16,079 DEBUG [RS:0;c42c682b34f7:45385 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c42c682b34f7,45385,1733812094403 2024-12-10T06:28:16,079 DEBUG [RS:2;c42c682b34f7:35565 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c42c682b34f7,35565,1733812094566 2024-12-10T06:28:16,080 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,41301,1733812094511' 2024-12-10T06:28:16,080 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,45385,1733812094403' 2024-12-10T06:28:16,080 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,35565,1733812094566' 2024-12-10T06:28:16,080 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T06:28:16,080 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T06:28:16,080 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T06:28:16,081 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T06:28:16,081 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T06:28:16,081 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T06:28:16,082 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T06:28:16,082 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T06:28:16,082 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T06:28:16,082 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T06:28:16,082 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T06:28:16,082 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T06:28:16,082 DEBUG [RS:0;c42c682b34f7:45385 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c42c682b34f7,45385,1733812094403 2024-12-10T06:28:16,082 DEBUG [RS:2;c42c682b34f7:35565 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c42c682b34f7,35565,1733812094566 2024-12-10T06:28:16,082 DEBUG [RS:1;c42c682b34f7:41301 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c42c682b34f7,41301,1733812094511 2024-12-10T06:28:16,082 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,35565,1733812094566' 2024-12-10T06:28:16,082 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,45385,1733812094403' 2024-12-10T06:28:16,082 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c42c682b34f7,41301,1733812094511' 2024-12-10T06:28:16,082 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T06:28:16,082 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T06:28:16,082 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T06:28:16,083 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T06:28:16,083 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T06:28:16,083 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T06:28:16,083 DEBUG [RS:0;c42c682b34f7:45385 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T06:28:16,084 INFO [RS:0;c42c682b34f7:45385 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T06:28:16,084 DEBUG [RS:2;c42c682b34f7:35565 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T06:28:16,084 INFO [RS:2;c42c682b34f7:35565 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T06:28:16,084 INFO [RS:0;c42c682b34f7:45385 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T06:28:16,084 DEBUG [RS:1;c42c682b34f7:41301 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T06:28:16,084 INFO [RS:2;c42c682b34f7:35565 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T06:28:16,084 INFO [RS:1;c42c682b34f7:41301 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T06:28:16,084 INFO [RS:1;c42c682b34f7:41301 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T06:28:16,189 INFO [RS:2;c42c682b34f7:35565 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:16,189 INFO [RS:1;c42c682b34f7:41301 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:16,189 INFO [RS:0;c42c682b34f7:45385 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:16,193 INFO [RS:0;c42c682b34f7:45385 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C45385%2C1733812094403, suffix=, logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403, archiveDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs, maxLogs=32 2024-12-10T06:28:16,193 INFO [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C41301%2C1733812094511, suffix=, logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511, archiveDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs, maxLogs=32 2024-12-10T06:28:16,193 INFO [RS:2;c42c682b34f7:35565 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C35565%2C1733812094566, suffix=, logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,35565,1733812094566, archiveDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs, maxLogs=32 2024-12-10T06:28:16,218 DEBUG [RS:2;c42c682b34f7:35565 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,35565,1733812094566/c42c682b34f7%2C35565%2C1733812094566.1733812096198, exclude list is [], retry=0 2024-12-10T06:28:16,220 DEBUG [RS:0;c42c682b34f7:45385 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403/c42c682b34f7%2C45385%2C1733812094403.1733812096198, exclude list is [], retry=0 2024-12-10T06:28:16,220 DEBUG [RS:1;c42c682b34f7:41301 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511/c42c682b34f7%2C41301%2C1733812094511.1733812096198, exclude list is [], retry=0 2024-12-10T06:28:16,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:16,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:16,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:16,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:16,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:16,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:16,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:16,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:16,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:16,237 INFO [RS:2;c42c682b34f7:35565 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,35565,1733812094566/c42c682b34f7%2C35565%2C1733812094566.1733812096198 2024-12-10T06:28:16,238 DEBUG [RS:2;c42c682b34f7:35565 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:16,241 INFO [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511/c42c682b34f7%2C41301%2C1733812094511.1733812096198 2024-12-10T06:28:16,241 DEBUG [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:16,242 INFO [RS:0;c42c682b34f7:45385 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403/c42c682b34f7%2C45385%2C1733812094403.1733812096198 2024-12-10T06:28:16,244 DEBUG [RS:0;c42c682b34f7:45385 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:16,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:16,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T06:28:16,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T06:28:16,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T06:28:16,266 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T06:28:16,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T06:28:16,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T06:28:16,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T06:28:16,274 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T06:28:16,274 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T06:28:16,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740 2024-12-10T06:28:16,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740 2024-12-10T06:28:16,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T06:28:16,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T06:28:16,280 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T06:28:16,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T06:28:16,286 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:16,287 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63259826, jitterRate=-0.057355135679244995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T06:28:16,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733812096257Initializing all the Stores at 1733812096259 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812096259Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812096259Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812096259Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812096259Cleaning up temporary data from old regions at 1733812096279 (+20 ms)Region opened successfully at 1733812096288 (+9 ms) 2024-12-10T06:28:16,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T06:28:16,288 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T06:28:16,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T06:28:16,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T06:28:16,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T06:28:16,289 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T06:28:16,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733812096288Disabling compacts and flushes for region at 1733812096288Disabling writes for close at 1733812096288Writing region close event to WAL at 1733812096289 (+1 ms)Closed at 1733812096289 2024-12-10T06:28:16,293 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T06:28:16,293 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T06:28:16,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T06:28:16,312 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T06:28:16,316 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T06:28:16,468 DEBUG [c42c682b34f7:34871 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-10T06:28:16,477 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(204): Hosts are {c42c682b34f7=0} racks are {/default-rack=0} 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-10T06:28:16,483 INFO [c42c682b34f7:34871 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T06:28:16,483 INFO [c42c682b34f7:34871 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T06:28:16,483 INFO [c42c682b34f7:34871 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-10T06:28:16,483 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T06:28:16,491 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:16,496 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c42c682b34f7,41301,1733812094511, state=OPENING 2024-12-10T06:28:16,518 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T06:28:16,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:16,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:16,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:16,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:16,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,537 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T06:28:16,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c42c682b34f7,41301,1733812094511}] 2024-12-10T06:28:16,715 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T06:28:16,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49275, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T06:28:16,730 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T06:28:16,731 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:16,731 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T06:28:16,735 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c42c682b34f7%2C41301%2C1733812094511.meta, suffix=.meta, logDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511, archiveDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs, maxLogs=32 2024-12-10T06:28:16,751 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511/c42c682b34f7%2C41301%2C1733812094511.meta.1733812096736.meta, exclude list is [], retry=0 2024-12-10T06:28:16,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:16,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:16,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:16,759 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,41301,1733812094511/c42c682b34f7%2C41301%2C1733812094511.meta.1733812096736.meta 2024-12-10T06:28:16,760 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:16,761 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:16,763 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T06:28:16,767 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T06:28:16,773 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T06:28:16,779 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T06:28:16,780 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:16,780 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T06:28:16,780 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T06:28:16,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T06:28:16,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T06:28:16,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T06:28:16,787 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T06:28:16,788 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,788 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T06:28:16,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T06:28:16,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T06:28:16,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T06:28:16,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:16,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T06:28:16,793 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T06:28:16,794 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740 2024-12-10T06:28:16,797 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740 2024-12-10T06:28:16,800 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T06:28:16,800 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T06:28:16,801 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T06:28:16,803 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T06:28:16,805 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65291551, jitterRate=-0.027080073952674866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T06:28:16,805 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T06:28:16,807 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733812096781Writing region info on filesystem at 1733812096781Initializing all the Stores at 1733812096783 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812096783Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812096784 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812096784Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733812096784Cleaning up temporary data from old regions at 1733812096800 (+16 ms)Running coprocessor post-open hooks at 1733812096805 (+5 ms)Region opened successfully at 1733812096807 (+2 ms) 2024-12-10T06:28:16,817 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733812096706 2024-12-10T06:28:16,830 DEBUG [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T06:28:16,831 INFO [RS_OPEN_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T06:28:16,832 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:16,834 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c42c682b34f7,41301,1733812094511, state=OPEN 2024-12-10T06:28:16,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T06:28:16,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T06:28:16,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T06:28:16,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T06:28:16,850 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,850 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,850 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,850 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T06:28:16,850 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:16,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T06:28:16,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c42c682b34f7,41301,1733812094511 in 311 msec 2024-12-10T06:28:16,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T06:28:16,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 556 msec 2024-12-10T06:28:16,866 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T06:28:16,866 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T06:28:16,888 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:28:16,889 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,41301,1733812094511, seqNum=-1] 2024-12-10T06:28:16,911 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:28:16,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38165, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:28:16,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2510 sec 2024-12-10T06:28:16,933 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733812096933, completionTime=-1 2024-12-10T06:28:16,937 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-10T06:28:16,938 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T06:28:16,970 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-10T06:28:16,971 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733812156971 2024-12-10T06:28:16,971 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733812216971 2024-12-10T06:28:16,971 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-10T06:28:16,973 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T06:28:16,981 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,34871,1733812093474-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,982 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,34871,1733812093474-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,982 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,34871,1733812093474-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,984 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c42c682b34f7:34871, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,985 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,985 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:16,991 DEBUG [master/c42c682b34f7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T06:28:17,017 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.304sec 2024-12-10T06:28:17,019 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T06:28:17,020 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T06:28:17,021 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T06:28:17,022 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T06:28:17,022 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T06:28:17,023 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,34871,1733812093474-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T06:28:17,023 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,34871,1733812093474-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T06:28:17,028 DEBUG [master/c42c682b34f7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T06:28:17,029 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T06:28:17,029 INFO [master/c42c682b34f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c42c682b34f7,34871,1733812093474-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:17,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78619a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:28:17,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c42c682b34f7,34871,-1 for getting cluster id 2024-12-10T06:28:17,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T06:28:17,052 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2c533b3b-6bbc-4d1b-8169-651bba667a42' 2024-12-10T06:28:17,055 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T06:28:17,055 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2c533b3b-6bbc-4d1b-8169-651bba667a42" 2024-12-10T06:28:17,058 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f79812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:28:17,058 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c42c682b34f7,34871,-1] 2024-12-10T06:28:17,060 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T06:28:17,063 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:17,063 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T06:28:17,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd02f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T06:28:17,066 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T06:28:17,073 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c42c682b34f7,41301,1733812094511, seqNum=-1] 2024-12-10T06:28:17,073 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:28:17,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:28:17,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c42c682b34f7,34871,1733812093474 2024-12-10T06:28:17,130 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40333/hbase 2024-12-10T06:28:17,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=6195 2024-12-10T06:28:17,177 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:17,180 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:17,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:17,188 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-84222190, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-84222190, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:17,204 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-84222190/hregion-84222190.1733812097189, exclude list is [], retry=0 2024-12-10T06:28:17,209 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:17,209 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:17,210 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:17,216 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-84222190/hregion-84222190.1733812097189 2024-12-10T06:28:17,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:17,218 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cd2bb665e40770e5bf7d7c141184b59c, NAME => 'testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:17,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741838_1014 (size=64) 2024-12-10T06:28:17,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741838_1014 (size=64) 2024-12-10T06:28:17,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741838_1014 (size=64) 2024-12-10T06:28:17,234 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:17,236 INFO [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,241 INFO [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd2bb665e40770e5bf7d7c141184b59c columnFamilyName a 2024-12-10T06:28:17,241 DEBUG [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:17,242 INFO [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] regionserver.HStore(327): Store=cd2bb665e40770e5bf7d7c141184b59c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:17,243 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,244 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,246 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,246 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,249 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,253 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:17,254 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cd2bb665e40770e5bf7d7c141184b59c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71919209, jitterRate=0.0716797262430191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:17,256 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cd2bb665e40770e5bf7d7c141184b59c: Writing region info on filesystem at 1733812097234Initializing all the Stores at 1733812097235 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812097235Cleaning up temporary data from old regions at 1733812097246 (+11 ms)Region opened successfully at 1733812097256 (+10 ms) 2024-12-10T06:28:17,256 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cd2bb665e40770e5bf7d7c141184b59c, disabling compactions & flushes 2024-12-10T06:28:17,256 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c. 2024-12-10T06:28:17,256 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c. 2024-12-10T06:28:17,256 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c. after waiting 0 ms 2024-12-10T06:28:17,256 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c. 2024-12-10T06:28:17,257 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c. 2024-12-10T06:28:17,257 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cd2bb665e40770e5bf7d7c141184b59c: Waiting for close lock at 1733812097256Disabling compacts and flushes for region at 1733812097256Disabling writes for close at 1733812097256Writing region close event to WAL at 1733812097257 (+1 ms)Closed at 1733812097257 2024-12-10T06:28:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741837_1013 (size=93) 2024-12-10T06:28:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741837_1013 (size=93) 2024-12-10T06:28:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741837_1013 (size=93) 2024-12-10T06:28:17,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:17,271 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-84222190:(num 1733812097189) 2024-12-10T06:28:17,273 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T06:28:17,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741839_1015 (size=276) 2024-12-10T06:28:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741839_1015 (size=276) 2024-12-10T06:28:17,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741839_1015 (size=276) 2024-12-10T06:28:17,301 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T06:28:17,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741840_1016 (size=230) 2024-12-10T06:28:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741840_1016 (size=230) 2024-12-10T06:28:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741840_1016 (size=230) 2024-12-10T06:28:17,351 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1, size=276 (276bytes) 2024-12-10T06:28:17,352 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-10T06:28:17,352 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-10T06:28:17,352 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1 2024-12-10T06:28:17,358 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1 after 4ms 2024-12-10T06:28:17,365 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:17,366 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1 took 16ms 2024-12-10T06:28:17,371 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1 so closing down 2024-12-10T06:28:17,377 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:17,378 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-10T06:28:17,381 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000001-wal-1.temp 2024-12-10T06:28:17,382 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:17,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741841_1017 (size=276) 2024-12-10T06:28:17,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741841_1017 (size=276) 2024-12-10T06:28:17,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741841_1017 (size=276) 2024-12-10T06:28:17,414 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:17,417 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002 2024-12-10T06:28:17,422 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-10T06:28:17,422 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1, journal: Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1, size=276 (276bytes) at 1733812097351Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1 so closing down at 1733812097371 (+20 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000001-wal-1.temp at 1733812097381 (+10 ms)3 split writer threads finished at 1733812097382 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733812097414 (+32 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002 at 1733812097417 (+3 ms)Processed 2 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733812097422 (+5 ms) 2024-12-10T06:28:17,441 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2, size=230 (230bytes) 2024-12-10T06:28:17,441 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2 2024-12-10T06:28:17,442 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2 after 1ms 2024-12-10T06:28:17,446 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:17,447 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2 took 7ms 2024-12-10T06:28:17,449 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2 so closing down 2024-12-10T06:28:17,449 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:17,452 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-10T06:28:17,453 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002-wal-2.temp 2024-12-10T06:28:17,454 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:17,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741842_1018 (size=230) 2024-12-10T06:28:17,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741842_1018 (size=230) 2024-12-10T06:28:17,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741842_1018 (size=230) 2024-12-10T06:28:17,463 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:17,467 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:17,469 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-10T06:28:17,471 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-10T06:28:17,471 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2, journal: Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2, size=230 (230bytes) at 1733812097441Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2 so closing down at 1733812097449 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002-wal-2.temp at 1733812097453 (+4 ms)3 split writer threads finished at 1733812097454 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733812097463 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733812097471 (+8 ms) 2024-12-10T06:28:17,471 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:17,474 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:17,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal.1733812097474, exclude list is [], retry=0 2024-12-10T06:28:17,492 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:17,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:17,494 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:17,498 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal.1733812097474 2024-12-10T06:28:17,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:17,499 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cd2bb665e40770e5bf7d7c141184b59c, NAME => 'testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:17,499 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:17,500 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,500 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,502 INFO [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,504 INFO [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd2bb665e40770e5bf7d7c141184b59c columnFamilyName a 2024-12-10T06:28:17,504 DEBUG [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:17,505 INFO [StoreOpener-cd2bb665e40770e5bf7d7c141184b59c-1 {}] regionserver.HStore(327): Store=cd2bb665e40770e5bf7d7c141184b59c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:17,505 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,506 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,510 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,512 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002 2024-12-10T06:28:17,516 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:17,525 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002 2024-12-10T06:28:17,529 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cd2bb665e40770e5bf7d7c141184b59c 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-10T06:28:17,593 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/.tmp/a/ce6f9c66a8c34c9ebc6ee94d6a4649d3 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733812097271/Put/seqid=0 2024-12-10T06:28:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741844_1020 (size=5170) 2024-12-10T06:28:17,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741844_1020 (size=5170) 2024-12-10T06:28:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741844_1020 (size=5170) 2024-12-10T06:28:17,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/.tmp/a/ce6f9c66a8c34c9ebc6ee94d6a4649d3 2024-12-10T06:28:17,674 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/.tmp/a/ce6f9c66a8c34c9ebc6ee94d6a4649d3 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/a/ce6f9c66a8c34c9ebc6ee94d6a4649d3 2024-12-10T06:28:17,686 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/a/ce6f9c66a8c34c9ebc6ee94d6a4649d3, entries=2, sequenceid=2, filesize=5.0 K 2024-12-10T06:28:17,693 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for cd2bb665e40770e5bf7d7c141184b59c in 163ms, sequenceid=2, compaction requested=false; wal=null 2024-12-10T06:28:17,695 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/0000000000000000002 2024-12-10T06:28:17,696 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,696 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,706 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cd2bb665e40770e5bf7d7c141184b59c 2024-12-10T06:28:17,711 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/cd2bb665e40770e5bf7d7c141184b59c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-10T06:28:17,713 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cd2bb665e40770e5bf7d7c141184b59c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72436029, jitterRate=0.07938094437122345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:17,714 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cd2bb665e40770e5bf7d7c141184b59c: Writing region info on filesystem at 1733812097500Initializing all the Stores at 1733812097502 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812097502Obtaining lock to block concurrent updates at 1733812097529 (+27 ms)Preparing flush snapshotting stores in cd2bb665e40770e5bf7d7c141184b59c at 1733812097529Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733812097534 (+5 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733812097178.cd2bb665e40770e5bf7d7c141184b59c. at 1733812097534Flushing cd2bb665e40770e5bf7d7c141184b59c/a: creating writer at 1733812097536 (+2 ms)Flushing cd2bb665e40770e5bf7d7c141184b59c/a: appending metadata at 1733812097581 (+45 ms)Flushing cd2bb665e40770e5bf7d7c141184b59c/a: closing flushed file at 1733812097583 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c324612: reopening flushed file at 1733812097671 (+88 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for cd2bb665e40770e5bf7d7c141184b59c in 163ms, sequenceid=2, compaction requested=false; wal=null at 1733812097693 (+22 ms)Cleaning up temporary data from old regions at 1733812097696 (+3 ms)Region opened successfully at 1733812097714 (+18 ms) 2024-12-10T06:28:17,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=374 (was 363) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:55890 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:48984 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:55730 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40333/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:48820 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:46228 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:46098 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6166 (was 6195) 2024-12-10T06:28:17,769 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=6163 2024-12-10T06:28:17,790 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:17,793 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:17,794 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:17,801 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19783378, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-19783378, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:17,823 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19783378/hregion-19783378.1733812097802, exclude list is [], retry=0 2024-12-10T06:28:17,828 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:17,830 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:17,833 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:17,844 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19783378/hregion-19783378.1733812097802 2024-12-10T06:28:17,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:17,845 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0456d4fd5b2a394c27edf844a3475a65, NAME => 'testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:17,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741846_1022 (size=64) 2024-12-10T06:28:17,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741846_1022 (size=64) 2024-12-10T06:28:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741846_1022 (size=64) 2024-12-10T06:28:17,887 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:17,889 INFO [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,892 INFO [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0456d4fd5b2a394c27edf844a3475a65 columnFamilyName a 2024-12-10T06:28:17,892 DEBUG [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:17,893 INFO [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] regionserver.HStore(327): Store=0456d4fd5b2a394c27edf844a3475a65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:17,893 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,894 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,895 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,896 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,896 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,899 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:17,905 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:17,907 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0456d4fd5b2a394c27edf844a3475a65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65746532, jitterRate=-0.020300328731536865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:17,907 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0456d4fd5b2a394c27edf844a3475a65: Writing region info on filesystem at 1733812097887Initializing all the Stores at 1733812097889 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812097889Cleaning up temporary data from old regions at 1733812097896 (+7 ms)Region opened successfully at 1733812097907 (+11 ms) 2024-12-10T06:28:17,907 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0456d4fd5b2a394c27edf844a3475a65, disabling compactions & flushes 2024-12-10T06:28:17,907 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65. 2024-12-10T06:28:17,908 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65. 2024-12-10T06:28:17,908 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65. after waiting 0 ms 2024-12-10T06:28:17,908 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65. 2024-12-10T06:28:17,913 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65. 2024-12-10T06:28:17,913 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0456d4fd5b2a394c27edf844a3475a65: Waiting for close lock at 1733812097907Disabling compacts and flushes for region at 1733812097907Disabling writes for close at 1733812097908 (+1 ms)Writing region close event to WAL at 1733812097913 (+5 ms)Closed at 1733812097913 2024-12-10T06:28:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741845_1021 (size=93) 2024-12-10T06:28:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741845_1021 (size=93) 2024-12-10T06:28:17,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741845_1021 (size=93) 2024-12-10T06:28:17,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:17,932 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19783378:(num 1733812097802) 2024-12-10T06:28:17,934 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T06:28:17,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741847_1023 (size=276) 2024-12-10T06:28:17,976 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T06:28:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741847_1023 (size=276) 2024-12-10T06:28:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741847_1023 (size=276) 2024-12-10T06:28:18,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741848_1024 (size=230) 2024-12-10T06:28:18,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741848_1024 (size=230) 2024-12-10T06:28:18,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741848_1024 (size=230) 2024-12-10T06:28:18,038 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2, size=230 (230bytes) 2024-12-10T06:28:18,038 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2 2024-12-10T06:28:18,040 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2 after 2ms 2024-12-10T06:28:18,045 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:18,045 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2 took 8ms 2024-12-10T06:28:18,054 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2 so closing down 2024-12-10T06:28:18,054 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:18,057 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-10T06:28:18,059 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002-wal-2.temp 2024-12-10T06:28:18,059 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:18,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741849_1025 (size=230) 2024-12-10T06:28:18,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741849_1025 (size=230) 2024-12-10T06:28:18,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741849_1025 (size=230) 2024-12-10T06:28:18,080 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:18,083 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 2024-12-10T06:28:18,083 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-10T06:28:18,084 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2, journal: Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2, size=230 (230bytes) at 1733812098038Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2 so closing down at 1733812098054 (+16 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002-wal-2.temp at 1733812098059 (+5 ms)3 split writer threads finished at 1733812098059Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733812098081 (+22 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 at 1733812098083 (+2 ms)Processed 1 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733812098083 2024-12-10T06:28:18,102 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1, size=276 (276bytes) 2024-12-10T06:28:18,102 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1 2024-12-10T06:28:18,103 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1 after 1ms 2024-12-10T06:28:18,108 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:18,108 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1 took 7ms 2024-12-10T06:28:18,125 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1 so closing down 2024-12-10T06:28:18,125 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:18,128 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-10T06:28:18,132 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000001-wal-1.temp 2024-12-10T06:28:18,134 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:18,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741850_1026 (size=276) 2024-12-10T06:28:18,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741850_1026 (size=276) 2024-12-10T06:28:18,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741850_1026 (size=276) 2024-12-10T06:28:18,157 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:18,163 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:18,166 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002, length=230 2024-12-10T06:28:18,171 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 2024-12-10T06:28:18,172 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 52 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-10T06:28:18,172 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1, journal: Splitting hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1, size=276 (276bytes) at 1733812098102Finishing writing output for hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1 so closing down at 1733812098125 (+23 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000001-wal-1.temp at 1733812098132 (+7 ms)3 split writer threads finished at 1733812098134 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733812098157 (+23 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 at 1733812098171 (+14 ms)Processed 2 edits across 1 Regions in 52 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733812098172 (+1 ms) 2024-12-10T06:28:18,172 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:18,175 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:18,197 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal.1733812098176, exclude list is [], retry=0 2024-12-10T06:28:18,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:18,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:18,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:18,217 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal.1733812098176 2024-12-10T06:28:18,220 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:18,221 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0456d4fd5b2a394c27edf844a3475a65, NAME => 'testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:18,221 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:18,221 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,221 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,224 INFO [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,228 INFO [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0456d4fd5b2a394c27edf844a3475a65 columnFamilyName a 2024-12-10T06:28:18,229 DEBUG [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:18,229 INFO [StoreOpener-0456d4fd5b2a394c27edf844a3475a65-1 {}] regionserver.HStore(327): Store=0456d4fd5b2a394c27edf844a3475a65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:18,230 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,233 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,234 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 2024-12-10T06:28:18,238 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:18,243 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 2024-12-10T06:28:18,243 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0456d4fd5b2a394c27edf844a3475a65 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-10T06:28:18,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/.tmp/a/f9e418adecbf45d88946b8d02cc7c9bc is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733812097932/Put/seqid=0 2024-12-10T06:28:18,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741852_1028 (size=5170) 2024-12-10T06:28:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741852_1028 (size=5170) 2024-12-10T06:28:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741852_1028 (size=5170) 2024-12-10T06:28:18,686 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/.tmp/a/f9e418adecbf45d88946b8d02cc7c9bc 2024-12-10T06:28:18,696 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/.tmp/a/f9e418adecbf45d88946b8d02cc7c9bc as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/a/f9e418adecbf45d88946b8d02cc7c9bc 2024-12-10T06:28:18,704 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/a/f9e418adecbf45d88946b8d02cc7c9bc, entries=2, sequenceid=2, filesize=5.0 K 2024-12-10T06:28:18,704 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0456d4fd5b2a394c27edf844a3475a65 in 461ms, sequenceid=2, compaction requested=false; wal=null 2024-12-10T06:28:18,705 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/0000000000000000002 2024-12-10T06:28:18,705 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,705 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,708 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0456d4fd5b2a394c27edf844a3475a65 2024-12-10T06:28:18,711 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/0456d4fd5b2a394c27edf844a3475a65/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-10T06:28:18,713 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0456d4fd5b2a394c27edf844a3475a65; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74338222, jitterRate=0.1077258288860321}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:18,713 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0456d4fd5b2a394c27edf844a3475a65: Writing region info on filesystem at 1733812098221Initializing all the Stores at 1733812098223 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812098223Obtaining lock to block concurrent updates at 1733812098243 (+20 ms)Preparing flush snapshotting stores in 0456d4fd5b2a394c27edf844a3475a65 at 1733812098243Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733812098243Flushing stores of testReplayEditsWrittenIntoWAL,,1733812097791.0456d4fd5b2a394c27edf844a3475a65. at 1733812098243Flushing 0456d4fd5b2a394c27edf844a3475a65/a: creating writer at 1733812098243Flushing 0456d4fd5b2a394c27edf844a3475a65/a: appending metadata at 1733812098266 (+23 ms)Flushing 0456d4fd5b2a394c27edf844a3475a65/a: closing flushed file at 1733812098266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d3248e6: reopening flushed file at 1733812098694 (+428 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0456d4fd5b2a394c27edf844a3475a65 in 461ms, sequenceid=2, compaction requested=false; wal=null at 1733812098704 (+10 ms)Cleaning up temporary data from old regions at 1733812098705 (+1 ms)Region opened successfully at 1733812098713 (+8 ms) 2024-12-10T06:28:18,738 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:46312 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:49052 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:55730 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:55984 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:46098 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 292), ProcessCount=11 (was 11), AvailableMemoryMB=6079 (was 6163) 2024-12-10T06:28:18,754 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=6077 2024-12-10T06:28:18,777 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:18,779 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:18,780 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:18,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80461513, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-80461513, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:18,798 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80461513/hregion-80461513.1733812098785, exclude list is [], retry=0 2024-12-10T06:28:18,802 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:18,802 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:18,803 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:18,806 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80461513/hregion-80461513.1733812098785 2024-12-10T06:28:18,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:18,807 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => affb646160e94cf4918eaa4c903f170a, NAME => 'testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:18,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741854_1030 (size=64) 2024-12-10T06:28:18,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741854_1030 (size=64) 2024-12-10T06:28:18,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741854_1030 (size=64) 2024-12-10T06:28:18,819 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:18,821 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,823 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region affb646160e94cf4918eaa4c903f170a columnFamilyName a 2024-12-10T06:28:18,823 DEBUG [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:18,823 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(327): Store=affb646160e94cf4918eaa4c903f170a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:18,823 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,825 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region affb646160e94cf4918eaa4c903f170a columnFamilyName b 2024-12-10T06:28:18,825 DEBUG [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:18,826 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(327): Store=affb646160e94cf4918eaa4c903f170a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:18,826 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,828 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region affb646160e94cf4918eaa4c903f170a columnFamilyName c 2024-12-10T06:28:18,828 DEBUG [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:18,828 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(327): Store=affb646160e94cf4918eaa4c903f170a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:18,829 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,829 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,830 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,831 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,831 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,832 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:18,833 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:18,835 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:18,836 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened affb646160e94cf4918eaa4c903f170a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74996256, jitterRate=0.11753129959106445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:18,836 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for affb646160e94cf4918eaa4c903f170a: Writing region info on filesystem at 1733812098819Initializing all the Stores at 1733812098820 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812098820Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812098821 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812098821Cleaning up temporary data from old regions at 1733812098831 (+10 ms)Region opened successfully at 1733812098836 (+5 ms) 2024-12-10T06:28:18,837 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing affb646160e94cf4918eaa4c903f170a, disabling compactions & flushes 2024-12-10T06:28:18,837 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:18,837 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:18,837 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. after waiting 0 ms 2024-12-10T06:28:18,837 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:18,837 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:18,837 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for affb646160e94cf4918eaa4c903f170a: Waiting for close lock at 1733812098836Disabling compacts and flushes for region at 1733812098836Disabling writes for close at 1733812098837 (+1 ms)Writing region close event to WAL at 1733812098837Closed at 1733812098837 2024-12-10T06:28:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741853_1029 (size=93) 2024-12-10T06:28:18,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741853_1029 (size=93) 2024-12-10T06:28:18,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741853_1029 (size=93) 2024-12-10T06:28:18,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:18,844 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80461513:(num 1733812098785) 2024-12-10T06:28:18,844 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:18,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:18,861 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848, exclude list is [], retry=0 2024-12-10T06:28:18,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:18,866 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:18,866 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:18,868 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 2024-12-10T06:28:18,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:19,079 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848, size=0 (0bytes) 2024-12-10T06:28:19,079 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 might be still open, length is 0 2024-12-10T06:28:19,079 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 2024-12-10T06:28:19,081 WARN [IPC Server handler 3 on default port 40333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-10T06:28:19,081 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 after 2ms 2024-12-10T06:28:21,517 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:49080 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33683:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49080 dst: /127.0.0.1:33683 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33683 remote=/127.0.0.1:49080]. Total timeout mills is 60000, 57515 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:21,518 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:56020 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56020 dst: /127.0.0.1:45191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:21,518 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:46342 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46342 dst: /127.0.0.1:40347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741855_1032 (size=200589) 2024-12-10T06:28:21,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741855_1032 (size=200589) 2024-12-10T06:28:21,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741855_1032 (size=200589) 2024-12-10T06:28:22,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:28:22,333 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T06:28:23,082 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 after 4003ms 2024-12-10T06:28:23,090 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:23,094 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 took 4015ms 2024-12-10T06:28:23,100 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733812098848.temp 2024-12-10T06:28:23,106 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000000001-wal.1733812098848.temp 2024-12-10T06:28:23,219 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848; continuing. 2024-12-10T06:28:23,220 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 so closing down 2024-12-10T06:28:23,220 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:23,220 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:23,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741856_1033 (size=200597) 2024-12-10T06:28:23,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741856_1033 (size=200597) 2024-12-10T06:28:23,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741856_1033 (size=200597) 2024-12-10T06:28:23,226 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000000001-wal.1733812098848.temp (wrote 3002 edits, skipped 0 edits in 47 ms) 2024-12-10T06:28:23,228 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000000001-wal.1733812098848.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002 2024-12-10T06:28:23,229 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 135 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T06:28:23,229 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848, journal: Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848, size=0 (0bytes) at 1733812099079Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000000001-wal.1733812098848.temp at 1733812103106 (+4027 ms)Split 1024 edits, skipped 0 edits. at 1733812103165 (+59 ms)Split 2048 edits, skipped 0 edits. at 1733812103192 (+27 ms)Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 so closing down at 1733812103220 (+28 ms)3 split writer threads finished at 1733812103220Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000000001-wal.1733812098848.temp (wrote 3002 edits, skipped 0 edits in 47 ms) at 1733812103226 (+6 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000000001-wal.1733812098848.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002 at 1733812103229 (+3 ms)Processed 3002 edits across 1 Regions in 135 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848, size=0, length=0, corrupted=false, cancelled=false at 1733812103229 2024-12-10T06:28:23,232 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812098848 2024-12-10T06:28:23,234 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002 2024-12-10T06:28:23,235 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:23,238 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:23,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812103239, exclude list is [], retry=0 2024-12-10T06:28:23,264 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:23,264 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:23,265 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:23,269 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812103239 2024-12-10T06:28:23,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:23,271 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:23,276 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:23,277 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region affb646160e94cf4918eaa4c903f170a columnFamilyName a 2024-12-10T06:28:23,278 DEBUG [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:23,278 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(327): Store=affb646160e94cf4918eaa4c903f170a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:23,278 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:23,279 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region affb646160e94cf4918eaa4c903f170a columnFamilyName b 2024-12-10T06:28:23,279 DEBUG [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:23,280 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(327): Store=affb646160e94cf4918eaa4c903f170a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:23,280 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:23,281 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region affb646160e94cf4918eaa4c903f170a columnFamilyName c 2024-12-10T06:28:23,281 DEBUG [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:23,282 INFO [StoreOpener-affb646160e94cf4918eaa4c903f170a-1 {}] regionserver.HStore(327): Store=affb646160e94cf4918eaa4c903f170a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:23,282 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:23,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:23,285 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:23,286 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002 2024-12-10T06:28:23,289 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:23,329 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T06:28:23,719 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing affb646160e94cf4918eaa4c903f170a 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T06:28:23,756 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/a/759ed99f13044dfea0a72c21b45470e8 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733812098875/Put/seqid=0 2024-12-10T06:28:23,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741858_1035 (size=50463) 2024-12-10T06:28:23,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741858_1035 (size=50463) 2024-12-10T06:28:23,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741858_1035 (size=50463) 2024-12-10T06:28:23,769 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/a/759ed99f13044dfea0a72c21b45470e8 2024-12-10T06:28:23,777 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/a/759ed99f13044dfea0a72c21b45470e8 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/a/759ed99f13044dfea0a72c21b45470e8 2024-12-10T06:28:23,786 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/a/759ed99f13044dfea0a72c21b45470e8, entries=754, sequenceid=754, filesize=49.3 K 2024-12-10T06:28:23,786 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for affb646160e94cf4918eaa4c903f170a in 67ms, sequenceid=754, compaction requested=false; wal=null 2024-12-10T06:28:23,813 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T06:28:23,814 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing affb646160e94cf4918eaa4c903f170a 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T06:28:23,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/a/b53b0d8c3b8b411ab56f8ed7931aacf1 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733812098911/Put/seqid=0 2024-12-10T06:28:23,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741859_1036 (size=20072) 2024-12-10T06:28:23,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741859_1036 (size=20072) 2024-12-10T06:28:23,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741859_1036 (size=20072) 2024-12-10T06:28:23,833 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/a/b53b0d8c3b8b411ab56f8ed7931aacf1 2024-12-10T06:28:23,859 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/b/d2caf60c3a9647b98ed494483a4bf2d0 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733812098931/Put/seqid=0 2024-12-10T06:28:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741860_1037 (size=35835) 2024-12-10T06:28:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741860_1037 (size=35835) 2024-12-10T06:28:23,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741860_1037 (size=35835) 2024-12-10T06:28:23,869 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/b/d2caf60c3a9647b98ed494483a4bf2d0 2024-12-10T06:28:23,877 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/a/b53b0d8c3b8b411ab56f8ed7931aacf1 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/a/b53b0d8c3b8b411ab56f8ed7931aacf1 2024-12-10T06:28:23,884 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/a/b53b0d8c3b8b411ab56f8ed7931aacf1, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-10T06:28:23,885 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/b/d2caf60c3a9647b98ed494483a4bf2d0 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/b/d2caf60c3a9647b98ed494483a4bf2d0 2024-12-10T06:28:23,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T06:28:23,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:23,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T06:28:23,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T06:28:23,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T06:28:23,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:23,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-10T06:28:23,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:23,893 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/b/d2caf60c3a9647b98ed494483a4bf2d0, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-10T06:28:23,894 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for affb646160e94cf4918eaa4c903f170a in 81ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-10T06:28:23,903 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T06:28:23,903 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing affb646160e94cf4918eaa4c903f170a 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T06:28:23,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/b/ba0df27ca80045468e0396f7d815d9e5 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733812098957/Put/seqid=0 2024-12-10T06:28:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741861_1038 (size=35082) 2024-12-10T06:28:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741861_1038 (size=35082) 2024-12-10T06:28:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741861_1038 (size=35082) 2024-12-10T06:28:23,919 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/b/ba0df27ca80045468e0396f7d815d9e5 2024-12-10T06:28:23,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/c/8fc21a64d96544ffaef19ed9c801ab93 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733812098982/Put/seqid=0 2024-12-10T06:28:23,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741862_1039 (size=20825) 2024-12-10T06:28:23,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741862_1039 (size=20825) 2024-12-10T06:28:23,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741862_1039 (size=20825) 2024-12-10T06:28:23,955 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/c/8fc21a64d96544ffaef19ed9c801ab93 2024-12-10T06:28:23,962 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/b/ba0df27ca80045468e0396f7d815d9e5 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/b/ba0df27ca80045468e0396f7d815d9e5 2024-12-10T06:28:23,969 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/b/ba0df27ca80045468e0396f7d815d9e5, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-10T06:28:23,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/c/8fc21a64d96544ffaef19ed9c801ab93 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/c/8fc21a64d96544ffaef19ed9c801ab93 2024-12-10T06:28:23,977 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/c/8fc21a64d96544ffaef19ed9c801ab93, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-10T06:28:23,977 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for affb646160e94cf4918eaa4c903f170a in 74ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-10T06:28:23,990 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733812099030/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:23,994 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002 2024-12-10T06:28:23,995 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T06:28:23,995 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing affb646160e94cf4918eaa4c903f170a 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-10T06:28:24,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/c/45b1f7f59252419aa8a7af0e5150da6c is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733812098990/Put/seqid=0 2024-12-10T06:28:24,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741863_1040 (size=50301) 2024-12-10T06:28:24,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741863_1040 (size=50301) 2024-12-10T06:28:24,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741863_1040 (size=50301) 2024-12-10T06:28:24,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/c/45b1f7f59252419aa8a7af0e5150da6c 2024-12-10T06:28:24,021 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45b1f7f59252419aa8a7af0e5150da6c 2024-12-10T06:28:24,022 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/.tmp/c/45b1f7f59252419aa8a7af0e5150da6c as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/c/45b1f7f59252419aa8a7af0e5150da6c 2024-12-10T06:28:24,029 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45b1f7f59252419aa8a7af0e5150da6c 2024-12-10T06:28:24,029 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/c/45b1f7f59252419aa8a7af0e5150da6c, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-10T06:28:24,030 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for affb646160e94cf4918eaa4c903f170a in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-10T06:28:24,031 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/0000000000000003002 2024-12-10T06:28:24,032 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:24,032 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:24,033 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T06:28:24,034 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for affb646160e94cf4918eaa4c903f170a 2024-12-10T06:28:24,037 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenIntoWAL/affb646160e94cf4918eaa4c903f170a/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-10T06:28:24,038 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened affb646160e94cf4918eaa4c903f170a; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72362415, jitterRate=0.07828401029109955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T06:28:24,038 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for affb646160e94cf4918eaa4c903f170a: Writing region info on filesystem at 1733812103271Initializing all the Stores at 1733812103273 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812103273Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812103276 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812103276Cleaning up temporary data from old regions at 1733812104032 (+756 ms)Region opened successfully at 1733812104038 (+6 ms) 2024-12-10T06:28:24,093 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing affb646160e94cf4918eaa4c903f170a, disabling compactions & flushes 2024-12-10T06:28:24,093 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:24,093 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:24,093 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. after waiting 0 ms 2024-12-10T06:28:24,093 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:24,095 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733812098777.affb646160e94cf4918eaa4c903f170a. 2024-12-10T06:28:24,095 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for affb646160e94cf4918eaa4c903f170a: Waiting for close lock at 1733812104093Disabling compacts and flushes for region at 1733812104093Disabling writes for close at 1733812104093Writing region close event to WAL at 1733812104095 (+2 ms)Closed at 1733812104095 2024-12-10T06:28:24,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741857_1034 (size=93) 2024-12-10T06:28:24,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741857_1034 (size=93) 2024-12-10T06:28:24,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741857_1034 (size=93) 2024-12-10T06:28:24,103 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:24,103 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733812103239) 2024-12-10T06:28:24,119 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=398 (was 384) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726887361_22 at /127.0.0.1:59802 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40333 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726887361_22 at /127.0.0.1:48534 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726887361_22 at /127.0.0.1:33586 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:34845 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726887361_22 at /127.0.0.1:33598 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44593 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44593 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34845 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=293 (was 292) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5954 (was 6077) 2024-12-10T06:28:24,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=398, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=5953 2024-12-10T06:28:24,149 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:24,151 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:24,151 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:24,154 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21241658, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-21241658, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:24,169 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21241658/hregion-21241658.1733812104155, exclude list is [], retry=0 2024-12-10T06:28:24,172 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:24,172 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:24,173 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:24,178 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21241658/hregion-21241658.1733812104155 2024-12-10T06:28:24,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:24,180 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 318f93b3d0e7965ba4511721daf25aee, NAME => 'test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:24,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741865_1042 (size=43) 2024-12-10T06:28:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741865_1042 (size=43) 2024-12-10T06:28:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741865_1042 (size=43) 2024-12-10T06:28:24,197 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:24,198 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,200 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 318f93b3d0e7965ba4511721daf25aee columnFamilyName a 2024-12-10T06:28:24,200 DEBUG [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:24,200 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(327): Store=318f93b3d0e7965ba4511721daf25aee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:24,201 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,202 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 318f93b3d0e7965ba4511721daf25aee columnFamilyName b 2024-12-10T06:28:24,202 DEBUG [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:24,203 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(327): Store=318f93b3d0e7965ba4511721daf25aee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:24,203 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,204 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 318f93b3d0e7965ba4511721daf25aee columnFamilyName c 2024-12-10T06:28:24,204 DEBUG [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:24,205 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(327): Store=318f93b3d0e7965ba4511721daf25aee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:24,205 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,207 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,207 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,208 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:24,210 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:24,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 318f93b3d0e7965ba4511721daf25aee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72293101, jitterRate=0.07725115120410919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:24,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 318f93b3d0e7965ba4511721daf25aee: Writing region info on filesystem at 1733812104197Initializing all the Stores at 1733812104198 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812104198Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812104198Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812104198Cleaning up temporary data from old regions at 1733812104207 (+9 ms)Region opened successfully at 1733812104214 (+7 ms) 2024-12-10T06:28:24,214 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 318f93b3d0e7965ba4511721daf25aee, disabling compactions & flushes 2024-12-10T06:28:24,214 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:24,214 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:24,214 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. after waiting 0 ms 2024-12-10T06:28:24,214 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:24,215 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:24,215 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 318f93b3d0e7965ba4511721daf25aee: Waiting for close lock at 1733812104214Disabling compacts and flushes for region at 1733812104214Disabling writes for close at 1733812104214Writing region close event to WAL at 1733812104215 (+1 ms)Closed at 1733812104215 2024-12-10T06:28:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741864_1041 (size=93) 2024-12-10T06:28:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741864_1041 (size=93) 2024-12-10T06:28:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741864_1041 (size=93) 2024-12-10T06:28:24,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:24,222 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21241658:(num 1733812104155) 2024-12-10T06:28:24,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:24,224 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:24,241 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225, exclude list is [], retry=0 2024-12-10T06:28:24,245 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:24,245 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:24,246 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:24,248 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 2024-12-10T06:28:24,248 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:24,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741866_1043 (size=200357) 2024-12-10T06:28:24,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741866_1043 (size=200357) 2024-12-10T06:28:24,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741866_1043 (size=200357) 2024-12-10T06:28:24,419 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225, size=195.7 K (200357bytes) 2024-12-10T06:28:24,419 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 2024-12-10T06:28:24,420 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 after 1ms 2024-12-10T06:28:24,423 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:24,425 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 took 6ms 2024-12-10T06:28:24,430 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733812104225.temp 2024-12-10T06:28:24,436 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000000001-wal.1733812104225.temp 2024-12-10T06:28:24,494 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 so closing down 2024-12-10T06:28:24,494 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:24,495 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:24,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741867_1044 (size=200357) 2024-12-10T06:28:24,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741867_1044 (size=200357) 2024-12-10T06:28:24,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741867_1044 (size=200357) 2024-12-10T06:28:24,501 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000000001-wal.1733812104225.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-12-10T06:28:24,503 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000000001-wal.1733812104225.temp to hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000 2024-12-10T06:28:24,503 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-10T06:28:24,504 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225, journal: Splitting hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225, size=195.7 K (200357bytes) at 1733812104419Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000000001-wal.1733812104225.temp at 1733812104436 (+17 ms)Split 1024 edits, skipped 0 edits. at 1733812104451 (+15 ms)Split 2048 edits, skipped 0 edits. at 1733812104473 (+22 ms)Finishing writing output for hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 so closing down at 1733812104494 (+21 ms)3 split writer threads finished at 1733812104495 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000000001-wal.1733812104225.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1733812104501 (+6 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000000001-wal.1733812104225.temp to hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000 at 1733812104503 (+2 ms)Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733812104503 2024-12-10T06:28:24,506 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104225 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812104225 2024-12-10T06:28:24,507 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000 2024-12-10T06:28:24,507 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:24,509 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:24,524 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510, exclude list is [], retry=0 2024-12-10T06:28:24,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:24,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:24,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:24,530 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 2024-12-10T06:28:24,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:24,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741868_1045 (size=200484) 2024-12-10T06:28:24,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741868_1045 (size=200484) 2024-12-10T06:28:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741868_1045 (size=200484) 2024-12-10T06:28:24,706 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510, size=195.8 K (200484bytes) 2024-12-10T06:28:24,706 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 2024-12-10T06:28:24,706 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 after 0ms 2024-12-10T06:28:24,709 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:24,711 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 took 6ms 2024-12-10T06:28:24,716 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733812104510.temp 2024-12-10T06:28:24,718 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003001-wal.1733812104510.temp 2024-12-10T06:28:24,769 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 so closing down 2024-12-10T06:28:24,769 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:24,770 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741869_1046 (size=200484) 2024-12-10T06:28:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741869_1046 (size=200484) 2024-12-10T06:28:24,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741869_1046 (size=200484) 2024-12-10T06:28:24,774 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003001-wal.1733812104510.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-12-10T06:28:24,776 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003001-wal.1733812104510.temp to hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000 2024-12-10T06:28:24,776 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-10T06:28:24,776 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510, journal: Splitting hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510, size=195.8 K (200484bytes) at 1733812104706Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003001-wal.1733812104510.temp at 1733812104718 (+12 ms)Split 1024 edits, skipped 0 edits. at 1733812104731 (+13 ms)Split 2048 edits, skipped 0 edits. at 1733812104751 (+20 ms)Finishing writing output for hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 so closing down at 1733812104769 (+18 ms)3 split writer threads finished at 1733812104770 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003001-wal.1733812104510.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1733812104774 (+4 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003001-wal.1733812104510.temp to hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000 at 1733812104776 (+2 ms)Processed 3000 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733812104776 2024-12-10T06:28:24,778 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104510 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812104510 2024-12-10T06:28:24,779 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000 2024-12-10T06:28:24,780 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:24,782 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/test2727-manual,16010,1733812104148, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:24,797 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104782, exclude list is [], retry=0 2024-12-10T06:28:24,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:24,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:24,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:24,805 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733812104148/wal.1733812104782 2024-12-10T06:28:24,806 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:24,806 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 318f93b3d0e7965ba4511721daf25aee, NAME => 'test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:24,806 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:24,806 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,807 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,808 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,810 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 318f93b3d0e7965ba4511721daf25aee columnFamilyName a 2024-12-10T06:28:24,810 DEBUG [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:24,810 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(327): Store=318f93b3d0e7965ba4511721daf25aee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:24,811 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,812 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 318f93b3d0e7965ba4511721daf25aee columnFamilyName b 2024-12-10T06:28:24,812 DEBUG [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:24,812 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(327): Store=318f93b3d0e7965ba4511721daf25aee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:24,813 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,814 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 318f93b3d0e7965ba4511721daf25aee columnFamilyName c 2024-12-10T06:28:24,814 DEBUG [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:24,814 INFO [StoreOpener-318f93b3d0e7965ba4511721daf25aee-1 {}] regionserver.HStore(327): Store=318f93b3d0e7965ba4511721daf25aee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:24,815 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,815 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:24,818 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000 2024-12-10T06:28:24,821 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:24,867 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000 2024-12-10T06:28:24,869 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000 2024-12-10T06:28:24,872 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:24,925 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000 2024-12-10T06:28:24,925 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 318f93b3d0e7965ba4511721daf25aee 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-10T06:28:24,959 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/a/952d2e3728db41f0875a66edc0d9c949 is 41, key is test2727/a:100/1733812104536/Put/seqid=0 2024-12-10T06:28:24,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741871_1048 (size=84227) 2024-12-10T06:28:24,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741871_1048 (size=84227) 2024-12-10T06:28:24,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741871_1048 (size=84227) 2024-12-10T06:28:24,973 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/a/952d2e3728db41f0875a66edc0d9c949 2024-12-10T06:28:25,014 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/b/4f6016b6f2a54aefa3a7ea78407e4edc is 41, key is test2727/b:100/1733812104587/Put/seqid=0 2024-12-10T06:28:25,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741872_1049 (size=84609) 2024-12-10T06:28:25,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741872_1049 (size=84609) 2024-12-10T06:28:25,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741872_1049 (size=84609) 2024-12-10T06:28:25,026 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/b/4f6016b6f2a54aefa3a7ea78407e4edc 2024-12-10T06:28:25,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/c/91f96fa13f584d6383f6c75b7e3965b9 is 41, key is test2727/c:100/1733812104635/Put/seqid=0 2024-12-10T06:28:25,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741873_1050 (size=84609) 2024-12-10T06:28:25,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741873_1050 (size=84609) 2024-12-10T06:28:25,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741873_1050 (size=84609) 2024-12-10T06:28:25,065 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/c/91f96fa13f584d6383f6c75b7e3965b9 2024-12-10T06:28:25,072 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/a/952d2e3728db41f0875a66edc0d9c949 as hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/a/952d2e3728db41f0875a66edc0d9c949 2024-12-10T06:28:25,080 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/a/952d2e3728db41f0875a66edc0d9c949, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-10T06:28:25,081 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/b/4f6016b6f2a54aefa3a7ea78407e4edc as hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/b/4f6016b6f2a54aefa3a7ea78407e4edc 2024-12-10T06:28:25,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/b/4f6016b6f2a54aefa3a7ea78407e4edc, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-10T06:28:25,091 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/.tmp/c/91f96fa13f584d6383f6c75b7e3965b9 as hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/c/91f96fa13f584d6383f6c75b7e3965b9 2024-12-10T06:28:25,098 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/c/91f96fa13f584d6383f6c75b7e3965b9, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-10T06:28:25,099 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 318f93b3d0e7965ba4511721daf25aee in 174ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-10T06:28:25,100 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000003000 2024-12-10T06:28:25,100 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/0000000000000006000 2024-12-10T06:28:25,101 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:25,101 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:25,102 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:25,104 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 318f93b3d0e7965ba4511721daf25aee 2024-12-10T06:28:25,107 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/test2727/318f93b3d0e7965ba4511721daf25aee/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-10T06:28:25,108 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 318f93b3d0e7965ba4511721daf25aee; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70036754, jitterRate=0.043628960847854614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:25,109 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 318f93b3d0e7965ba4511721daf25aee: Writing region info on filesystem at 1733812104807Initializing all the Stores at 1733812104808 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812104808Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812104808Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812104808Obtaining lock to block concurrent updates at 1733812104925 (+117 ms)Preparing flush snapshotting stores in 318f93b3d0e7965ba4511721daf25aee at 1733812104925Finished memstore snapshotting test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733812104925Flushing stores of test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. at 1733812104925Flushing 318f93b3d0e7965ba4511721daf25aee/a: creating writer at 1733812104925Flushing 318f93b3d0e7965ba4511721daf25aee/a: appending metadata at 1733812104958 (+33 ms)Flushing 318f93b3d0e7965ba4511721daf25aee/a: closing flushed file at 1733812104958Flushing 318f93b3d0e7965ba4511721daf25aee/b: creating writer at 1733812104980 (+22 ms)Flushing 318f93b3d0e7965ba4511721daf25aee/b: appending metadata at 1733812105011 (+31 ms)Flushing 318f93b3d0e7965ba4511721daf25aee/b: closing flushed file at 1733812105012 (+1 ms)Flushing 318f93b3d0e7965ba4511721daf25aee/c: creating writer at 1733812105034 (+22 ms)Flushing 318f93b3d0e7965ba4511721daf25aee/c: appending metadata at 1733812105054 (+20 ms)Flushing 318f93b3d0e7965ba4511721daf25aee/c: closing flushed file at 1733812105054Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40353c4a: reopening flushed file at 1733812105071 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b0b12a: reopening flushed file at 1733812105080 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4feb33c7: reopening flushed file at 1733812105090 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 318f93b3d0e7965ba4511721daf25aee in 174ms, sequenceid=6000, compaction requested=false; wal=null at 1733812105099 (+9 ms)Cleaning up temporary data from old regions at 1733812105101 (+2 ms)Region opened successfully at 1733812105109 (+8 ms) 2024-12-10T06:28:25,111 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-10T06:28:25,111 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 318f93b3d0e7965ba4511721daf25aee, disabling compactions & flushes 2024-12-10T06:28:25,111 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:25,111 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:25,111 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. after waiting 0 ms 2024-12-10T06:28:25,111 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:25,113 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733812104149.318f93b3d0e7965ba4511721daf25aee. 2024-12-10T06:28:25,113 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 318f93b3d0e7965ba4511721daf25aee: Waiting for close lock at 1733812105111Disabling compacts and flushes for region at 1733812105111Disabling writes for close at 1733812105111Writing region close event to WAL at 1733812105113 (+2 ms)Closed at 1733812105113 2024-12-10T06:28:25,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741870_1047 (size=93) 2024-12-10T06:28:25,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741870_1047 (size=93) 2024-12-10T06:28:25,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741870_1047 (size=93) 2024-12-10T06:28:25,120 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:25,120 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733812104782) 2024-12-10T06:28:25,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=402 (was 398) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:48672 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:33598 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:59912 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=293 (was 293), ProcessCount=11 (was 11), AvailableMemoryMB=5750 (was 5953) 2024-12-10T06:28:25,148 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=5749 2024-12-10T06:28:25,165 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:25,172 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:25,173 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733812105173 2024-12-10T06:28:25,180 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173 2024-12-10T06:28:25,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:25,186 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c1abd1309394d3c955898d2e499746e, NAME => 'testSequentialEditLogSeqNum,,1733812105166.7c1abd1309394d3c955898d2e499746e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:25,186 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733812105166.7c1abd1309394d3c955898d2e499746e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:25,186 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,186 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,187 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e doesn't exist for region: 7c1abd1309394d3c955898d2e499746e on table testSequentialEditLogSeqNum 2024-12-10T06:28:25,188 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 7c1abd1309394d3c955898d2e499746e on table testSequentialEditLogSeqNum 2024-12-10T06:28:25,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741875_1052 (size=62) 2024-12-10T06:28:25,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741875_1052 (size=62) 2024-12-10T06:28:25,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741875_1052 (size=62) 2024-12-10T06:28:25,203 INFO [StoreOpener-7c1abd1309394d3c955898d2e499746e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,205 INFO [StoreOpener-7c1abd1309394d3c955898d2e499746e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c1abd1309394d3c955898d2e499746e columnFamilyName a 2024-12-10T06:28:25,205 DEBUG [StoreOpener-7c1abd1309394d3c955898d2e499746e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,206 INFO [StoreOpener-7c1abd1309394d3c955898d2e499746e-1 {}] regionserver.HStore(327): Store=7c1abd1309394d3c955898d2e499746e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,206 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,207 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,208 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,209 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7c1abd1309394d3c955898d2e499746e 2024-12-10T06:28:25,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:25,212 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7c1abd1309394d3c955898d2e499746e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67599532, jitterRate=0.007311522960662842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:25,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7c1abd1309394d3c955898d2e499746e: Writing region info on filesystem at 1733812105186Initializing all the Stores at 1733812105203 (+17 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105203Cleaning up temporary data from old regions at 1733812105208 (+5 ms)Region opened successfully at 1733812105214 (+6 ms) 2024-12-10T06:28:25,227 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7c1abd1309394d3c955898d2e499746e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-10T06:28:25,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/.tmp/a/01cf76b2c81a47c0adda96af671fdb20 is 81, key is testSequentialEditLogSeqNum/a:x0/1733812105214/Put/seqid=0 2024-12-10T06:28:25,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741876_1053 (size=5833) 2024-12-10T06:28:25,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741876_1053 (size=5833) 2024-12-10T06:28:25,260 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/.tmp/a/01cf76b2c81a47c0adda96af671fdb20 2024-12-10T06:28:25,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741876_1053 (size=5833) 2024-12-10T06:28:25,267 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/.tmp/a/01cf76b2c81a47c0adda96af671fdb20 as hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/a/01cf76b2c81a47c0adda96af671fdb20 2024-12-10T06:28:25,274 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/a/01cf76b2c81a47c0adda96af671fdb20, entries=10, sequenceid=13, filesize=5.7 K 2024-12-10T06:28:25,276 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7c1abd1309394d3c955898d2e499746e in 49ms, sequenceid=13, compaction requested=false 2024-12-10T06:28:25,276 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7c1abd1309394d3c955898d2e499746e: 2024-12-10T06:28:25,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T06:28:25,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T06:28:25,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T06:28:25,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T06:28:25,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T06:28:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741874_1051 (size=1616) 2024-12-10T06:28:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741874_1051 (size=1616) 2024-12-10T06:28:25,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741874_1051 (size=1616) 2024-12-10T06:28:25,305 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173, size=1.6 K (1616bytes) 2024-12-10T06:28:25,305 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173 2024-12-10T06:28:25,305 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173 after 0ms 2024-12-10T06:28:25,308 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:25,309 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173 took 5ms 2024-12-10T06:28:25,311 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173 so closing down 2024-12-10T06:28:25,312 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:25,313 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733812105173.temp 2024-12-10T06:28:25,314 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000003-wal.1733812105173.temp 2024-12-10T06:28:25,315 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:25,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741877_1054 (size=1175) 2024-12-10T06:28:25,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741877_1054 (size=1175) 2024-12-10T06:28:25,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741877_1054 (size=1175) 2024-12-10T06:28:25,327 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000003-wal.1733812105173.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:25,329 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000003-wal.1733812105173.temp to hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000020 2024-12-10T06:28:25,329 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-10T06:28:25,329 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173, journal: Splitting hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173, size=1.6 K (1616bytes) at 1733812105305Finishing writing output for hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173 so closing down at 1733812105312 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000003-wal.1733812105173.temp at 1733812105314 (+2 ms)3 split writer threads finished at 1733812105315 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000003-wal.1733812105173.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733812105327 (+12 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000003-wal.1733812105173.temp to hdfs://localhost:40333/hbase/data/default/testSequentialEditLogSeqNum/7c1abd1309394d3c955898d2e499746e/recovered.edits/0000000000000000020 at 1733812105329 (+2 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:40333/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733812105164/wal.1733812105173, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733812105329 2024-12-10T06:28:25,346 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:33598 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:59912 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=293 (was 293), ProcessCount=11 (was 11), AvailableMemoryMB=5730 (was 5749) 2024-12-10T06:28:25,359 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=5729 2024-12-10T06:28:25,376 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:25,379 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:25,401 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:25,404 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98256046, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-98256046, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:25,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98256046/hregion-98256046.1733812105405, exclude list is [], retry=0 2024-12-10T06:28:25,426 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:25,426 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:25,427 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:25,430 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98256046/hregion-98256046.1733812105405 2024-12-10T06:28:25,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:25,431 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cb157a562ac87dc2ee2cd43bc1785149, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:25,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741879_1056 (size=70) 2024-12-10T06:28:25,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741879_1056 (size=70) 2024-12-10T06:28:25,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741879_1056 (size=70) 2024-12-10T06:28:25,446 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:25,448 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,450 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName a 2024-12-10T06:28:25,450 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,451 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,451 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,453 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName b 2024-12-10T06:28:25,453 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,454 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,454 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,456 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName c 2024-12-10T06:28:25,456 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,456 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,456 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,457 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,458 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,459 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,459 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,460 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:25,461 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,463 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:25,464 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb157a562ac87dc2ee2cd43bc1785149; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62048577, jitterRate=-0.07540415227413177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:25,465 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb157a562ac87dc2ee2cd43bc1785149: Writing region info on filesystem at 1733812105446Initializing all the Stores at 1733812105448 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105448Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105448Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105448Cleaning up temporary data from old regions at 1733812105459 (+11 ms)Region opened successfully at 1733812105465 (+6 ms) 2024-12-10T06:28:25,465 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cb157a562ac87dc2ee2cd43bc1785149, disabling compactions & flushes 2024-12-10T06:28:25,465 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:25,465 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:25,465 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. after waiting 0 ms 2024-12-10T06:28:25,465 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:25,466 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:25,466 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cb157a562ac87dc2ee2cd43bc1785149: Waiting for close lock at 1733812105465Disabling compacts and flushes for region at 1733812105465Disabling writes for close at 1733812105465Writing region close event to WAL at 1733812105466 (+1 ms)Closed at 1733812105466 2024-12-10T06:28:25,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741878_1055 (size=93) 2024-12-10T06:28:25,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741878_1055 (size=93) 2024-12-10T06:28:25,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741878_1055 (size=93) 2024-12-10T06:28:25,474 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:25,474 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98256046:(num 1733812105405) 2024-12-10T06:28:25,474 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:25,476 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:25,521 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477, exclude list is [], retry=0 2024-12-10T06:28:25,524 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:25,525 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:25,525 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:25,527 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 2024-12-10T06:28:25,528 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:25,528 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cb157a562ac87dc2ee2cd43bc1785149, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:25,528 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:25,528 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,529 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,531 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,532 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName a 2024-12-10T06:28:25,532 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,533 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,533 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,534 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName b 2024-12-10T06:28:25,534 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,535 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,535 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,536 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName c 2024-12-10T06:28:25,536 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:25,536 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:25,537 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,537 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,538 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,539 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,539 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,540 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:25,542 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:25,543 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb157a562ac87dc2ee2cd43bc1785149; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64883695, jitterRate=-0.033157601952552795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:25,544 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb157a562ac87dc2ee2cd43bc1785149: Writing region info on filesystem at 1733812105529Initializing all the Stores at 1733812105530 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105530Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105530Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812105530Cleaning up temporary data from old regions at 1733812105539 (+9 ms)Region opened successfully at 1733812105543 (+4 ms) 2024-12-10T06:28:25,548 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733812105547/Put/seqid=0 2024-12-10T06:28:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741881_1058 (size=4826) 2024-12-10T06:28:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741881_1058 (size=4826) 2024-12-10T06:28:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741881_1058 (size=4826) 2024-12-10T06:28:25,557 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40333/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in cb157a562ac87dc2ee2cd43bc1785149/a 2024-12-10T06:28:25,565 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-10T06:28:25,566 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T06:28:25,566 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cb157a562ac87dc2ee2cd43bc1785149: 2024-12-10T06:28:25,568 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_ 2024-12-10T06:28:25,570 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40333/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into cb157a562ac87dc2ee2cd43bc1785149/a as hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_ - updating store file list. 2024-12-10T06:28:25,576 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:25,576 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_ into cb157a562ac87dc2ee2cd43bc1785149/a 2024-12-10T06:28:25,576 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40333/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into cb157a562ac87dc2ee2cd43bc1785149/a (new location: hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_) 2024-12-10T06:28:25,619 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477, size=0 (0bytes) 2024-12-10T06:28:25,619 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 might be still open, length is 0 2024-12-10T06:28:25,619 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 2024-12-10T06:28:25,620 WARN [IPC Server handler 1 on default port 40333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-10T06:28:25,620 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 after 0ms 2024-12-10T06:28:27,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:60006 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33683:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60006 dst: /127.0.0.1:33683 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33683 remote=/127.0.0.1:60006]. Total timeout mills is 60000, 58073 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:27,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:33800 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33800 dst: /127.0.0.1:45191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:27,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:48794 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48794 dst: /127.0.0.1:40347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:27,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741880_1059 (size=434) 2024-12-10T06:28:27,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741880_1059 (size=434) 2024-12-10T06:28:29,621 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 after 4001ms 2024-12-10T06:28:29,625 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:29,626 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 took 4007ms 2024-12-10T06:28:29,632 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477; continuing. 2024-12-10T06:28:29,632 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 so closing down 2024-12-10T06:28:29,632 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:29,634 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733812105477.temp 2024-12-10T06:28:29,641 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005-wal.1733812105477.temp 2024-12-10T06:28:29,641 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:29,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741882_1060 (size=236) 2024-12-10T06:28:29,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741882_1060 (size=236) 2024-12-10T06:28:29,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741882_1060 (size=236) 2024-12-10T06:28:29,657 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005-wal.1733812105477.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:29,658 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005-wal.1733812105477.temp to hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005 2024-12-10T06:28:29,659 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T06:28:29,659 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477, journal: Splitting hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477, size=0 (0bytes) at 1733812105619Finishing writing output for hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 so closing down at 1733812109632 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005-wal.1733812105477.temp at 1733812109641 (+9 ms)3 split writer threads finished at 1733812109641Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005-wal.1733812105477.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733812109657 (+16 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005-wal.1733812105477.temp to hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005 at 1733812109658 (+1 ms)Processed 2 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477, size=0, length=0, corrupted=false, cancelled=false at 1733812109659 (+1 ms) 2024-12-10T06:28:29,661 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812105477 2024-12-10T06:28:29,662 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005 2024-12-10T06:28:29,662 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:29,664 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:29,682 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812109665, exclude list is [], retry=0 2024-12-10T06:28:29,690 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:29,690 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:29,690 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:29,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812109665 2024-12-10T06:28:29,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:29,696 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cb157a562ac87dc2ee2cd43bc1785149, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:29,697 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:29,697 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,697 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,699 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,700 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName a 2024-12-10T06:28:29,700 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:29,706 DEBUG [StoreFileOpener-cb157a562ac87dc2ee2cd43bc1785149-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:29,706 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/6c332c9a71e44d1db87793d9cb55ae08_SeqId_3_ 2024-12-10T06:28:29,707 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:29,707 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,708 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName b 2024-12-10T06:28:29,708 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:29,708 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:29,709 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,709 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb157a562ac87dc2ee2cd43bc1785149 columnFamilyName c 2024-12-10T06:28:29,709 DEBUG [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:29,710 INFO [StoreOpener-cb157a562ac87dc2ee2cd43bc1785149-1 {}] regionserver.HStore(327): Store=cb157a562ac87dc2ee2cd43bc1785149/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:29,710 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,713 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005 2024-12-10T06:28:29,716 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:29,718 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005 2024-12-10T06:28:29,718 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb157a562ac87dc2ee2cd43bc1785149 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-10T06:28:29,741 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/.tmp/a/8b47a2d24d574d0f8a52ca04e73c68ad is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733812105582/Put/seqid=0 2024-12-10T06:28:29,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741884_1062 (size=5149) 2024-12-10T06:28:29,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741884_1062 (size=5149) 2024-12-10T06:28:29,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741884_1062 (size=5149) 2024-12-10T06:28:29,756 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/.tmp/a/8b47a2d24d574d0f8a52ca04e73c68ad 2024-12-10T06:28:29,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/.tmp/a/8b47a2d24d574d0f8a52ca04e73c68ad as hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/8b47a2d24d574d0f8a52ca04e73c68ad 2024-12-10T06:28:29,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/a/8b47a2d24d574d0f8a52ca04e73c68ad, entries=1, sequenceid=5, filesize=5.0 K 2024-12-10T06:28:29,769 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for cb157a562ac87dc2ee2cd43bc1785149 in 51ms, sequenceid=5, compaction requested=false; wal=null 2024-12-10T06:28:29,770 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/0000000000000000005 2024-12-10T06:28:29,771 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,771 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,772 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:29,774 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb157a562ac87dc2ee2cd43bc1785149 2024-12-10T06:28:29,777 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/cb157a562ac87dc2ee2cd43bc1785149/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-10T06:28:29,779 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb157a562ac87dc2ee2cd43bc1785149; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58880072, jitterRate=-0.12261855602264404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:29,780 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb157a562ac87dc2ee2cd43bc1785149: Writing region info on filesystem at 1733812109697Initializing all the Stores at 1733812109698 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812109698Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812109698Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812109699 (+1 ms)Obtaining lock to block concurrent updates at 1733812109718 (+19 ms)Preparing flush snapshotting stores in cb157a562ac87dc2ee2cd43bc1785149 at 1733812109718Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733812109718Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. at 1733812109718Flushing cb157a562ac87dc2ee2cd43bc1785149/a: creating writer at 1733812109718Flushing cb157a562ac87dc2ee2cd43bc1785149/a: appending metadata at 1733812109740 (+22 ms)Flushing cb157a562ac87dc2ee2cd43bc1785149/a: closing flushed file at 1733812109740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a5c1b14: reopening flushed file at 1733812109762 (+22 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for cb157a562ac87dc2ee2cd43bc1785149 in 51ms, sequenceid=5, compaction requested=false; wal=null at 1733812109769 (+7 ms)Cleaning up temporary data from old regions at 1733812109772 (+3 ms)Region opened successfully at 1733812109780 (+8 ms) 2024-12-10T06:28:29,784 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cb157a562ac87dc2ee2cd43bc1785149, disabling compactions & flushes 2024-12-10T06:28:29,784 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:29,785 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:29,785 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. after waiting 0 ms 2024-12-10T06:28:29,785 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:29,786 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733812105377.cb157a562ac87dc2ee2cd43bc1785149. 2024-12-10T06:28:29,786 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cb157a562ac87dc2ee2cd43bc1785149: Waiting for close lock at 1733812109784Disabling compacts and flushes for region at 1733812109784Disabling writes for close at 1733812109785 (+1 ms)Writing region close event to WAL at 1733812109786 (+1 ms)Closed at 1733812109786 2024-12-10T06:28:29,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741883_1061 (size=93) 2024-12-10T06:28:29,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741883_1061 (size=93) 2024-12-10T06:28:29,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741883_1061 (size=93) 2024-12-10T06:28:29,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:29,796 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733812109665) 2024-12-10T06:28:29,812 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=408 (was 407) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1819476332_22 at /127.0.0.1:60028 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40333 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1819476332_22 at /127.0.0.1:48824 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 293) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=6707 (was 5729) - AvailableMemoryMB LEAK? - 2024-12-10T06:28:29,828 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=408, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=6706 2024-12-10T06:28:29,852 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:29,859 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T06:28:29,864 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c42c682b34f7,34871,1733812093474 2024-12-10T06:28:29,867 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7217d84f 2024-12-10T06:28:29,868 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T06:28:29,870 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T06:28:29,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T06:28:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T06:28:29,884 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T06:28:29,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-10T06:28:29,886 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:29,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T06:28:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:28:29,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741885_1063 (size=694) 2024-12-10T06:28:29,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741885_1063 (size=694) 2024-12-10T06:28:29,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741885_1063 (size=694) 2024-12-10T06:28:29,920 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25 2024-12-10T06:28:29,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741886_1064 (size=77) 2024-12-10T06:28:29,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741886_1064 (size=77) 2024-12-10T06:28:29,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741886_1064 (size=77) 2024-12-10T06:28:29,941 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:29,942 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:29,942 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:29,942 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:29,942 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:29,942 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:29,942 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:29,942 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812109941Disabling compacts and flushes for region at 1733812109941Disabling writes for close at 1733812109942 (+1 ms)Writing region close event to WAL at 1733812109942Closed at 1733812109942 2024-12-10T06:28:29,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T06:28:29,952 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733812109946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733812109946"}]},"ts":"1733812109946"} 2024-12-10T06:28:29,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T06:28:29,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T06:28:29,966 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733812109963"}]},"ts":"1733812109963"} 2024-12-10T06:28:29,972 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-10T06:28:29,973 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c42c682b34f7=0} racks are {/default-rack=0} 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-10T06:28:29,975 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T06:28:29,975 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T06:28:29,975 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-10T06:28:29,975 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T06:28:29,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN}] 2024-12-10T06:28:29,979 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN 2024-12-10T06:28:29,981 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN; state=OFFLINE, location=c42c682b34f7,45385,1733812094403; forceNewPlan=false, retain=false 2024-12-10T06:28:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:28:30,135 INFO [c42c682b34f7:34871 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T06:28:30,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPENING, regionLocation=c42c682b34f7,45385,1733812094403 2024-12-10T06:28:30,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN because future has completed 2024-12-10T06:28:30,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403}] 2024-12-10T06:28:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:28:30,296 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T06:28:30,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56277, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T06:28:30,307 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,307 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:30,308 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,308 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:30,308 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,308 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,312 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,314 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf1 2024-12-10T06:28:30,314 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:30,315 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:30,315 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,317 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf2 2024-12-10T06:28:30,317 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:30,317 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:30,318 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,319 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,320 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,321 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,321 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,323 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T06:28:30,325 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,332 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:30,333 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8a37328b6903c9afef567df7be16e039; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70239927, jitterRate=0.046656474471092224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T06:28:30,333 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,334 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8a37328b6903c9afef567df7be16e039: Running coprocessor pre-open hook at 1733812110308Writing region info on filesystem at 1733812110308Initializing all the Stores at 1733812110311 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812110311Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812110311Cleaning up temporary data from old regions at 1733812110321 (+10 ms)Running coprocessor post-open hooks at 1733812110333 (+12 ms)Region opened successfully at 1733812110334 (+1 ms) 2024-12-10T06:28:30,337 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., pid=6, masterSystemTime=1733812110295 2024-12-10T06:28:30,344 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,344 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,345 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPEN, openSeqNum=2, regionLocation=c42c682b34f7,45385,1733812094403 2024-12-10T06:28:30,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403 because future has completed 2024-12-10T06:28:30,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T06:28:30,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403 in 210 msec 2024-12-10T06:28:30,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T06:28:30,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN in 378 msec 2024-12-10T06:28:30,360 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T06:28:30,360 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733812110360"}]},"ts":"1733812110360"} 2024-12-10T06:28:30,363 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-10T06:28:30,365 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T06:28:30,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 489 msec 2024-12-10T06:28:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T06:28:30,521 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-10T06:28:30,521 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-10T06:28:30,523 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T06:28:30,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-10T06:28:30,531 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T06:28:30,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-10T06:28:30,550 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=2] 2024-12-10T06:28:30,551 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T06:28:30,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T06:28:30,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8a37328b6903c9afef567df7be16e039, source=c42c682b34f7,45385,1733812094403, destination=c42c682b34f7,41301,1733812094511, warming up region on c42c682b34f7,41301,1733812094511 2024-12-10T06:28:30,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8a37328b6903c9afef567df7be16e039, source=c42c682b34f7,45385,1733812094403, destination=c42c682b34f7,41301,1733812094511, running balancer 2024-12-10T06:28:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE 2024-12-10T06:28:30,583 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE 2024-12-10T06:28:30,585 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=CLOSING, regionLocation=c42c682b34f7,45385,1733812094403 2024-12-10T06:28:30,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE because future has completed 2024-12-10T06:28:30,592 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:28:30,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403}] 2024-12-10T06:28:30,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:30,604 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,606 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf1 2024-12-10T06:28:30,606 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:30,607 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:30,607 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,609 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf2 2024-12-10T06:28:30,609 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:30,609 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:30,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812110610Disabling compacts and flushes for region at 1733812110610Disabling writes for close at 1733812110610Writing region close event to WAL at 1733812110610Closed at 1733812110610 2024-12-10T06:28:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T06:28:30,751 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,751 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:28:30,752 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:30,752 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,752 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,752 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:30,752 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,752 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 8a37328b6903c9afef567df7be16e039 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-10T06:28:30,770 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/44ffe5bac1c14685891e73877941b023 is 35, key is r1/cf1:q/1733812110559/Put/seqid=0 2024-12-10T06:28:30,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741887_1065 (size=4783) 2024-12-10T06:28:30,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741887_1065 (size=4783) 2024-12-10T06:28:30,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741887_1065 (size=4783) 2024-12-10T06:28:30,778 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/44ffe5bac1c14685891e73877941b023 2024-12-10T06:28:30,785 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/44ffe5bac1c14685891e73877941b023 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023 2024-12-10T06:28:30,791 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023, entries=1, sequenceid=5, filesize=4.7 K 2024-12-10T06:28:30,792 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 8a37328b6903c9afef567df7be16e039 in 40ms, sequenceid=5, compaction requested=false 2024-12-10T06:28:30,793 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-10T06:28:30,798 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-10T06:28:30,800 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:30,801 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812110752Running coprocessor pre-close hooks at 1733812110752Disabling compacts and flushes for region at 1733812110752Disabling writes for close at 1733812110752Obtaining lock to block concurrent updates at 1733812110752Preparing flush snapshotting stores in 8a37328b6903c9afef567df7be16e039 at 1733812110752Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733812110753 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. at 1733812110754 (+1 ms)Flushing 8a37328b6903c9afef567df7be16e039/cf1: creating writer at 1733812110754Flushing 8a37328b6903c9afef567df7be16e039/cf1: appending metadata at 1733812110770 (+16 ms)Flushing 8a37328b6903c9afef567df7be16e039/cf1: closing flushed file at 1733812110770Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78564bc6: reopening flushed file at 1733812110784 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 8a37328b6903c9afef567df7be16e039 in 40ms, sequenceid=5, compaction requested=false at 1733812110792 (+8 ms)Writing region close event to WAL at 1733812110794 (+2 ms)Running coprocessor post-close hooks at 1733812110798 (+4 ms)Closed at 1733812110800 (+2 ms) 2024-12-10T06:28:30,801 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8a37328b6903c9afef567df7be16e039 move to c42c682b34f7,41301,1733812094511 record at close sequenceid=5 2024-12-10T06:28:30,804 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:30,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=CLOSED 2024-12-10T06:28:30,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403 because future has completed 2024-12-10T06:28:30,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T06:28:30,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403 in 217 msec 2024-12-10T06:28:30,814 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE; state=CLOSED, location=c42c682b34f7,41301,1733812094511; forceNewPlan=false, retain=false 2024-12-10T06:28:30,965 INFO [c42c682b34f7:34871 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T06:28:30,966 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPENING, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:30,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE because future has completed 2024-12-10T06:28:30,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511}] 2024-12-10T06:28:31,129 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,129 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:31,129 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,129 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:31,130 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,130 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,131 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,133 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf1 2024-12-10T06:28:31,133 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:31,140 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023 2024-12-10T06:28:31,140 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:31,140 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,141 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf2 2024-12-10T06:28:31,141 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:31,141 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:31,142 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,142 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,144 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,145 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,145 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,146 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T06:28:31,148 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,148 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8a37328b6903c9afef567df7be16e039; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69037318, jitterRate=0.028736203908920288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T06:28:31,149 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,149 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8a37328b6903c9afef567df7be16e039: Running coprocessor pre-open hook at 1733812111130Writing region info on filesystem at 1733812111130Initializing all the Stores at 1733812111131 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812111131Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812111131Cleaning up temporary data from old regions at 1733812111145 (+14 ms)Running coprocessor post-open hooks at 1733812111149 (+4 ms)Region opened successfully at 1733812111149 2024-12-10T06:28:31,157 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., pid=9, masterSystemTime=1733812111124 2024-12-10T06:28:31,160 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,160 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPEN, openSeqNum=9, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:31,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511 because future has completed 2024-12-10T06:28:31,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T06:28:31,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511 in 194 msec 2024-12-10T06:28:31,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE in 587 msec 2024-12-10T06:28:31,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T06:28:31,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T06:28:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:35826 deadline: 1733812171195, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=41301 startCode=1733812094511. As of locationSeqNum=5. 2024-12-10T06:28:31,230 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=41301 startCode=1733812094511. As of locationSeqNum=5. 2024-12-10T06:28:31,231 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=41301 startCode=1733812094511. As of locationSeqNum=5. 2024-12-10T06:28:31,231 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,41301,1733812094511, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=41301 startCode=1733812094511. As of locationSeqNum=5. 2024-12-10T06:28:31,353 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8a37328b6903c9afef567df7be16e039 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-10T06:28:31,374 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/805786ae94c24567984474f96fcbcc53 is 29, key is r1/cf1:/1733812111341/DeleteFamily/seqid=0 2024-12-10T06:28:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741888_1066 (size=4906) 2024-12-10T06:28:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741888_1066 (size=4906) 2024-12-10T06:28:31,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741888_1066 (size=4906) 2024-12-10T06:28:31,386 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,392 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,408 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf2/5ecf67f58d8c42bb868857a1abab599a is 29, key is r1/cf2:/1733812111341/DeleteFamily/seqid=0 2024-12-10T06:28:31,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741889_1067 (size=4906) 2024-12-10T06:28:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741889_1067 (size=4906) 2024-12-10T06:28:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741889_1067 (size=4906) 2024-12-10T06:28:31,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf2/5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,425 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,426 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/805786ae94c24567984474f96fcbcc53 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,435 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,435 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53, entries=1, sequenceid=12, filesize=4.8 K 2024-12-10T06:28:31,436 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf2/5ecf67f58d8c42bb868857a1abab599a as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,445 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,445 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a, entries=1, sequenceid=12, filesize=4.8 K 2024-12-10T06:28:31,448 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 8a37328b6903c9afef567df7be16e039 in 96ms, sequenceid=12, compaction requested=false 2024-12-10T06:28:31,448 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8a37328b6903c9afef567df7be16e039: 2024-12-10T06:28:31,450 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T06:28:31,452 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8a37328b6903c9afef567df7be16e039/cf1 is initiating major compaction (all files) 2024-12-10T06:28:31,452 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T06:28:31,452 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:31,453 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8a37328b6903c9afef567df7be16e039/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,453 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023, hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53] into tmpdir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp, totalSize=9.5 K 2024-12-10T06:28:31,454 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 44ffe5bac1c14685891e73877941b023, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733812110559 2024-12-10T06:28:31,455 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 805786ae94c24567984474f96fcbcc53, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-10T06:28:31,467 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8a37328b6903c9afef567df7be16e039#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:28:31,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741890_1068 (size=4626) 2024-12-10T06:28:31,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741890_1068 (size=4626) 2024-12-10T06:28:31,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741890_1068 (size=4626) 2024-12-10T06:28:31,496 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf1/b684a2c7996746bf959df0168ac5502b as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/b684a2c7996746bf959df0168ac5502b 2024-12-10T06:28:31,517 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 8a37328b6903c9afef567df7be16e039/cf1 of 8a37328b6903c9afef567df7be16e039 into b684a2c7996746bf959df0168ac5502b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:28:31,518 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8a37328b6903c9afef567df7be16e039: 2024-12-10T06:28:31,518 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-10T06:28:31,518 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8a37328b6903c9afef567df7be16e039/cf2 is initiating major compaction (all files) 2024-12-10T06:28:31,518 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T06:28:31,518 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T06:28:31,518 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8a37328b6903c9afef567df7be16e039/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,519 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a] into tmpdir=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp, totalSize=4.8 K 2024-12-10T06:28:31,519 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5ecf67f58d8c42bb868857a1abab599a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-10T06:28:31,527 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8a37328b6903c9afef567df7be16e039#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T06:28:31,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741891_1069 (size=4592) 2024-12-10T06:28:31,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741891_1069 (size=4592) 2024-12-10T06:28:31,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741891_1069 (size=4592) 2024-12-10T06:28:31,553 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/.tmp/cf2/db5eadcfb50a48afb0e422b903de7053 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/db5eadcfb50a48afb0e422b903de7053 2024-12-10T06:28:31,562 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 8a37328b6903c9afef567df7be16e039/cf2 of 8a37328b6903c9afef567df7be16e039 into db5eadcfb50a48afb0e422b903de7053(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:28:31,562 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8a37328b6903c9afef567df7be16e039: 2024-12-10T06:28:31,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8a37328b6903c9afef567df7be16e039, source=c42c682b34f7,41301,1733812094511, destination=c42c682b34f7,45385,1733812094403, warming up region on c42c682b34f7,45385,1733812094403 2024-12-10T06:28:31,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8a37328b6903c9afef567df7be16e039, source=c42c682b34f7,41301,1733812094511, destination=c42c682b34f7,45385,1733812094403, running balancer 2024-12-10T06:28:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE 2024-12-10T06:28:31,571 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE 2024-12-10T06:28:31,572 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=CLOSING, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:31,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:31,575 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE because future has completed 2024-12-10T06:28:31,576 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T06:28:31,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511}] 2024-12-10T06:28:31,576 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf1 2024-12-10T06:28:31,576 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:31,586 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023 2024-12-10T06:28:31,592 INFO [StoreFileOpener-8a37328b6903c9afef567df7be16e039-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,592 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,598 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/b684a2c7996746bf959df0168ac5502b 2024-12-10T06:28:31,598 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:31,598 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,599 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf2 2024-12-10T06:28:31,600 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:31,612 INFO [StoreFileOpener-8a37328b6903c9afef567df7be16e039-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,612 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-10T06:28:31,623 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/db5eadcfb50a48afb0e422b903de7053 2024-12-10T06:28:31,623 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:31,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45385 {}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812111623Disabling compacts and flushes for region at 1733812111623Disabling writes for close at 1733812111624 (+1 ms)Writing region close event to WAL at 1733812111628 (+4 ms)Closed at 1733812111628 2024-12-10T06:28:31,731 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,731 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T06:28:31,732 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:31,732 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,732 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,732 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:31,732 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,732 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023, hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53] to archive 2024-12-10T06:28:31,737 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:28:31,742 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023 to hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/44ffe5bac1c14685891e73877941b023 2024-12-10T06:28:31,742 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53 to hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/805786ae94c24567984474f96fcbcc53 2024-12-10T06:28:31,766 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a] to archive 2024-12-10T06:28:31,767 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:28:31,770 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a to hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/5ecf67f58d8c42bb868857a1abab599a 2024-12-10T06:28:31,780 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-10T06:28:31,781 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:31,781 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812111731Running coprocessor pre-close hooks at 1733812111731Disabling compacts and flushes for region at 1733812111731Disabling writes for close at 1733812111732 (+1 ms)Writing region close event to WAL at 1733812111772 (+40 ms)Running coprocessor post-close hooks at 1733812111781 (+9 ms)Closed at 1733812111781 2024-12-10T06:28:31,781 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 8a37328b6903c9afef567df7be16e039 move to c42c682b34f7,45385,1733812094403 record at close sequenceid=12 2024-12-10T06:28:31,785 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:31,786 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=CLOSED 2024-12-10T06:28:31,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511 because future has completed 2024-12-10T06:28:31,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-10T06:28:31,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511 in 216 msec 2024-12-10T06:28:31,796 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE; state=CLOSED, location=c42c682b34f7,45385,1733812094403; forceNewPlan=false, retain=false 2024-12-10T06:28:31,946 INFO [c42c682b34f7:34871 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T06:28:31,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPENING, regionLocation=c42c682b34f7,45385,1733812094403 2024-12-10T06:28:31,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE because future has completed 2024-12-10T06:28:31,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403}] 2024-12-10T06:28:32,110 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,111 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:32,111 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,111 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:32,111 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,111 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,113 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,114 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf1 2024-12-10T06:28:32,114 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:32,119 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/b684a2c7996746bf959df0168ac5502b 2024-12-10T06:28:32,119 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:32,120 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,120 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf2 2024-12-10T06:28:32,121 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:32,126 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/db5eadcfb50a48afb0e422b903de7053 2024-12-10T06:28:32,126 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:32,126 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,127 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,128 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,129 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,129 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,130 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T06:28:32,131 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,133 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 8a37328b6903c9afef567df7be16e039; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64653201, jitterRate=-0.03659223020076752}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T06:28:32,133 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,133 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 8a37328b6903c9afef567df7be16e039: Running coprocessor pre-open hook at 1733812112111Writing region info on filesystem at 1733812112111Initializing all the Stores at 1733812112112 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812112112Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812112112Cleaning up temporary data from old regions at 1733812112129 (+17 ms)Running coprocessor post-open hooks at 1733812112133 (+4 ms)Region opened successfully at 1733812112133 2024-12-10T06:28:32,134 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., pid=12, masterSystemTime=1733812112106 2024-12-10T06:28:32,137 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,137 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,138 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPEN, openSeqNum=18, regionLocation=c42c682b34f7,45385,1733812094403 2024-12-10T06:28:32,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403 because future has completed 2024-12-10T06:28:32,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-10T06:28:32,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,45385,1733812094403 in 189 msec 2024-12-10T06:28:32,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, REOPEN/MOVE in 576 msec 2024-12-10T06:28:32,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T06:28:32,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T06:28:32,176 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server c42c682b34f7,45385,1733812094403: testing ***** 2024-12-10T06:28:32,176 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-10T06:28:32,179 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-10T06:28:32,181 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-10T06:28:32,184 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-10T06:28:32,185 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-10T06:28:32,193 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 315639008 }, "NonHeapMemoryUsage": { "committed": 172359680, "init": 7667712, "max": -1, "used": 169667752 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "c42c682b34f7", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "c42c682b34f7", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59695", "tag.serverName": "c42c682b34f7,41301,1733812094511", "tag.clusterId": "2c533b3b-6bbc-4d1b-8169-651bba667a42", "tag.Context": "regionserver", "tag.Hostname": "c42c682b34f7", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733812094511, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-10T06:28:32,197 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34871 {}] master.MasterRpcServices(700): c42c682b34f7,45385,1733812094403 reported a fatal error: ***** ABORTING region server c42c682b34f7,45385,1733812094403: testing ***** 2024-12-10T06:28:32,199 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c42c682b34f7,45385,1733812094403' ***** 2024-12-10T06:28:32,199 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-10T06:28:32,199 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T06:28:32,200 INFO [RS:0;c42c682b34f7:45385 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-10T06:28:32,200 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T06:28:32,200 INFO [RS:0;c42c682b34f7:45385 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-10T06:28:32,200 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41301 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:32792 deadline: 1733812172200, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=45385 startCode=1733812094403. As of locationSeqNum=12. 2024-12-10T06:28:32,200 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(956): aborting server c42c682b34f7,45385,1733812094403 2024-12-10T06:28:32,201 INFO [RS:0;c42c682b34f7:45385 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T06:28:32,201 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:32,201 INFO [RS:0;c42c682b34f7:45385 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c42c682b34f7:45385. 2024-12-10T06:28:32,201 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,41301,1733812094511, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,41301,1733812094511, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=45385 startCode=1733812094403. As of locationSeqNum=12. 2024-12-10T06:28:32,201 DEBUG [RS:0;c42c682b34f7:45385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:28:32,201 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,41301,1733812094511, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=45385 startCode=1733812094403. As of locationSeqNum=12. 2024-12-10T06:28:32,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,41301,1733812094511, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c42c682b34f7 port=45385 startCode=1733812094403. As of locationSeqNum=12. 2024-12-10T06:28:32,201 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:32,201 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,201 DEBUG [RS:0;c42c682b34f7:45385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:32,202 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T06:28:32,202 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1325): Online Regions={8a37328b6903c9afef567df7be16e039=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.} 2024-12-10T06:28:32,202 DEBUG [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1351): Waiting on 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:32,204 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,204 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812112201Running coprocessor pre-close hooks at 1733812112201Disabling compacts and flushes for region at 1733812112201Disabling writes for close at 1733812112201Writing region close event to WAL at 1733812112203 (+2 ms)Running coprocessor post-close hooks at 1733812112203Closed at 1733812112204 (+1 ms) 2024-12-10T06:28:32,204 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:32,241 INFO [regionserver/c42c682b34f7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:28:32,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c42c682b34f7,45385,1733812094403 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:32,311 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c42c682b34f7,45385,1733812094403 aborting 2024-12-10T06:28:32,311 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c42c682b34f7,45385,1733812094403 aborting 2024-12-10T06:28:32,311 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=12 from cache 2024-12-10T06:28:32,403 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(976): stopping server c42c682b34f7,45385,1733812094403; all regions closed. 2024-12-10T06:28:32,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741834_1010 (size=1677) 2024-12-10T06:28:32,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741834_1010 (size=1677) 2024-12-10T06:28:32,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741834_1010 (size=1677) 2024-12-10T06:28:32,408 DEBUG [RS:0;c42c682b34f7:45385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:32,408 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:28:32,408 INFO [RS:0;c42c682b34f7:45385 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T06:28:32,408 INFO [RS:0;c42c682b34f7:45385 {}] hbase.ChoreService(370): Chore service for: regionserver/c42c682b34f7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T06:28:32,408 INFO [regionserver/c42c682b34f7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T06:28:32,409 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T06:28:32,409 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T06:28:32,409 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T06:28:32,409 INFO [RS:0;c42c682b34f7:45385 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T06:28:32,409 INFO [RS:0;c42c682b34f7:45385 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45385 2024-12-10T06:28:32,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c42c682b34f7,45385,1733812094403 2024-12-10T06:28:32,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T06:28:32,428 INFO [RS:0;c42c682b34f7:45385 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T06:28:32,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c42c682b34f7,45385,1733812094403] 2024-12-10T06:28:32,457 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c42c682b34f7,45385,1733812094403 already deleted, retry=false 2024-12-10T06:28:32,457 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c42c682b34f7,45385,1733812094403 on c42c682b34f7,34871,1733812093474 2024-12-10T06:28:32,464 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c42c682b34f7,45385,1733812094403, splitWal=true, meta=false 2024-12-10T06:28:32,467 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for c42c682b34f7,45385,1733812094403 (carryingMeta=false) c42c682b34f7,45385,1733812094403/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@301ac937[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T06:28:32,467 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c42c682b34f7,45385,1733812094403, splitWal=true, meta=false 2024-12-10T06:28:32,469 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): c42c682b34f7,45385,1733812094403 had 1 regions 2024-12-10T06:28:32,470 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c42c682b34f7,45385,1733812094403, splitWal=true, meta=false, isMeta: false 2024-12-10T06:28:32,472 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting 2024-12-10T06:28:32,473 INFO [PEWorker-2 {}] master.SplitWALManager(105): c42c682b34f7,45385,1733812094403 WAL count=1, meta=false 2024-12-10T06:28:32,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c42c682b34f7%2C45385%2C1733812094403.1733812096198}] 2024-12-10T06:28:32,480 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:32,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c42c682b34f7%2C45385%2C1733812094403.1733812096198, worker=c42c682b34f7,41301,1733812094511}] 2024-12-10T06:28:32,523 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18] 2024-12-10T06:28:32,525 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c42c682b34f7:45385 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c42c682b34f7/172.17.0.2:45385 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:32,526 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18, error=java.net.ConnectException: Call to address=c42c682b34f7:45385 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c42c682b34f7/172.17.0.2:45385 2024-12-10T06:28:32,526 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-10T06:28:32,526 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18 from cache 2024-12-10T06:28:32,527 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address c42c682b34f7:45385 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c42c682b34f7/172.17.0.2:45385 2024-12-10T06:28:32,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:32,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45385-0x1019b0ee98c0001, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:32,543 INFO [RS:0;c42c682b34f7:45385 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T06:28:32,543 INFO [RS:0;c42c682b34f7:45385 {}] regionserver.HRegionServer(1031): Exiting; stopping=c42c682b34f7,45385,1733812094403; zookeeper connection closed. 2024-12-10T06:28:32,543 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@784f2a4b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@784f2a4b 2024-12-10T06:28:32,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41301 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-10T06:28:32,668 INFO [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198, size=1.6 K (1677bytes) 2024-12-10T06:28:32,669 INFO [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 2024-12-10T06:28:32,669 INFO [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 after 0ms 2024-12-10T06:28:32,672 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:32,672 INFO [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 took 4ms 2024-12-10T06:28:32,679 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 8a37328b6903c9afef567df7be16e039: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-10T06:28:32,680 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 so closing down 2024-12-10T06:28:32,680 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:32,680 INFO [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:32,680 INFO [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-10T06:28:32,680 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198, journal: Splitting hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198, size=1.6 K (1677bytes) at 1733812112669Finishing writing output for hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 so closing down at 1733812112680 (+11 ms)3 split writer threads finished at 1733812112680Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733812112680 2024-12-10T06:28:32,680 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 2024-12-10T06:28:32,682 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c42c682b34f7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-10T06:28:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34871 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-10T06:28:32,690 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting/c42c682b34f7%2C45385%2C1733812094403.1733812096198 to hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs 2024-12-10T06:28:32,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-10T06:28:32,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c42c682b34f7%2C45385%2C1733812094403.1733812096198, worker=c42c682b34f7,41301,1733812094511 in 208 msec 2024-12-10T06:28:32,694 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:32,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T06:28:32,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure c42c682b34f7%2C45385%2C1733812094403.1733812096198, worker=c42c682b34f7,41301,1733812094511 in 220 msec 2024-12-10T06:28:32,698 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting dir is empty, no logs to split. 2024-12-10T06:28:32,698 INFO [PEWorker-2 {}] master.SplitWALManager(105): c42c682b34f7,45385,1733812094403 WAL count=0, meta=false 2024-12-10T06:28:32,698 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c42c682b34f7,45385,1733812094403 WAL splitting is done? wals=0, meta=false 2024-12-10T06:28:32,700 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for c42c682b34f7,45385,1733812094403 failed, ignore...File hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/WALs/c42c682b34f7,45385,1733812094403-splitting does not exist. 2024-12-10T06:28:32,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN}] 2024-12-10T06:28:32,703 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN 2024-12-10T06:28:32,705 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T06:28:32,832 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18] 2024-12-10T06:28:32,833 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to c42c682b34f7:45385 this server is in the failed servers list 2024-12-10T06:28:32,834 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=c42c682b34f7:45385 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c42c682b34f7:45385 2024-12-10T06:28:32,834 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c42c682b34f7:45385 2024-12-10T06:28:32,834 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,45385,1733812094403, seqNum=18 from cache 2024-12-10T06:28:32,855 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(204): Hosts are {c42c682b34f7=0} racks are {/default-rack=0} 2024-12-10T06:28:32,855 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T06:28:32,855 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T06:28:32,855 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T06:28:32,855 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T06:28:32,855 INFO [c42c682b34f7:34871 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T06:28:32,855 INFO [c42c682b34f7:34871 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T06:28:32,855 DEBUG [c42c682b34f7:34871 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T06:28:32,856 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPENING, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:32,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN because future has completed 2024-12-10T06:28:32,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511}] 2024-12-10T06:28:33,017 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:33,018 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a37328b6903c9afef567df7be16e039, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:33,018 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,018 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:33,018 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,018 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,020 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,021 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf1 2024-12-10T06:28:33,022 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,031 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf1/b684a2c7996746bf959df0168ac5502b 2024-12-10T06:28:33,032 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,032 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,033 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a37328b6903c9afef567df7be16e039 columnFamilyName cf2 2024-12-10T06:28:33,033 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,040 DEBUG [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/cf2/db5eadcfb50a48afb0e422b903de7053 2024-12-10T06:28:33,040 INFO [StoreOpener-8a37328b6903c9afef567df7be16e039-1 {}] regionserver.HStore(327): Store=8a37328b6903c9afef567df7be16e039/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,041 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,042 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,043 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,044 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,045 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,045 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T06:28:33,047 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,048 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 8a37328b6903c9afef567df7be16e039; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62462530, jitterRate=-0.06923577189445496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T06:28:33,049 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:33,049 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 8a37328b6903c9afef567df7be16e039: Running coprocessor pre-open hook at 1733812113019Writing region info on filesystem at 1733812113019Initializing all the Stores at 1733812113020 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113020Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113020Cleaning up temporary data from old regions at 1733812113045 (+25 ms)Running coprocessor post-open hooks at 1733812113049 (+4 ms)Region opened successfully at 1733812113049 2024-12-10T06:28:33,051 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., pid=17, masterSystemTime=1733812113012 2024-12-10T06:28:33,054 DEBUG [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:33,054 INFO [RS_OPEN_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:33,055 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8a37328b6903c9afef567df7be16e039, regionState=OPEN, openSeqNum=18, regionLocation=c42c682b34f7,41301,1733812094511 2024-12-10T06:28:33,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511 because future has completed 2024-12-10T06:28:33,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-10T06:28:33,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a37328b6903c9afef567df7be16e039, server=c42c682b34f7,41301,1733812094511 in 208 msec 2024-12-10T06:28:33,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-10T06:28:33,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a37328b6903c9afef567df7be16e039, ASSIGN in 368 msec 2024-12-10T06:28:33,073 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server c42c682b34f7,45385,1733812094403 after splitting done 2024-12-10T06:28:33,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure c42c682b34f7,45385,1733812094403, splitWal=true, meta=false in 614 msec 2024-12-10T06:28:33,343 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., hostname=c42c682b34f7,41301,1733812094511, seqNum=18] 2024-12-10T06:28:33,358 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 408), OpenFileDescriptor=1045 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 317) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6377 (was 6706) 2024-12-10T06:28:33,360 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-12-10T06:28:33,376 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1045, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=6376 2024-12-10T06:28:33,376 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-12-10T06:28:33,399 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:33,401 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:33,402 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:33,406 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32259454, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-32259454, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:33,420 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32259454/hregion-32259454.1733812113406, exclude list is [], retry=0 2024-12-10T06:28:33,439 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:33,439 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:33,439 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:33,444 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32259454/hregion-32259454.1733812113406 2024-12-10T06:28:33,444 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:33,444 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c4214c320e558e88aac07b373bf22836, NAME => 'testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:33,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741893_1071 (size=67) 2024-12-10T06:28:33,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741893_1071 (size=67) 2024-12-10T06:28:33,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741893_1071 (size=67) 2024-12-10T06:28:33,456 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:33,457 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,476 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName a 2024-12-10T06:28:33,476 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,477 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,477 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,478 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName b 2024-12-10T06:28:33,478 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,479 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,479 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,480 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName c 2024-12-10T06:28:33,480 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,481 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,481 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,481 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,482 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,483 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,483 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,483 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:33,484 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,486 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:33,487 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4214c320e558e88aac07b373bf22836; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62602072, jitterRate=-0.06715643405914307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:33,487 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4214c320e558e88aac07b373bf22836: Writing region info on filesystem at 1733812113456Initializing all the Stores at 1733812113457 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113457Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113457Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113457Cleaning up temporary data from old regions at 1733812113483 (+26 ms)Region opened successfully at 1733812113487 (+4 ms) 2024-12-10T06:28:33,487 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c4214c320e558e88aac07b373bf22836, disabling compactions & flushes 2024-12-10T06:28:33,487 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,487 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,487 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. after waiting 0 ms 2024-12-10T06:28:33,487 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,488 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,488 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c4214c320e558e88aac07b373bf22836: Waiting for close lock at 1733812113487Disabling compacts and flushes for region at 1733812113487Disabling writes for close at 1733812113487Writing region close event to WAL at 1733812113488 (+1 ms)Closed at 1733812113488 2024-12-10T06:28:33,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741892_1070 (size=93) 2024-12-10T06:28:33,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741892_1070 (size=93) 2024-12-10T06:28:33,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741892_1070 (size=93) 2024-12-10T06:28:33,492 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:33,493 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32259454:(num 1733812113406) 2024-12-10T06:28:33,493 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:33,494 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:33,508 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494, exclude list is [], retry=0 2024-12-10T06:28:33,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:33,524 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:33,524 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:33,526 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 2024-12-10T06:28:33,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:33,526 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c4214c320e558e88aac07b373bf22836, NAME => 'testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:33,526 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:33,526 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,527 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,528 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,529 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName a 2024-12-10T06:28:33,529 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,529 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,529 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,530 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName b 2024-12-10T06:28:33,530 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,530 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,531 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,531 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName c 2024-12-10T06:28:33,531 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,532 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,532 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,532 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,534 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,534 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,535 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:33,536 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,537 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4214c320e558e88aac07b373bf22836; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64887138, jitterRate=-0.03310629725456238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:33,537 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4214c320e558e88aac07b373bf22836: Writing region info on filesystem at 1733812113527Initializing all the Stores at 1733812113527Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113527Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113528 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113528Cleaning up temporary data from old regions at 1733812113534 (+6 ms)Region opened successfully at 1733812113537 (+3 ms) 2024-12-10T06:28:33,594 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c4214c320e558e88aac07b373bf22836 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-10T06:28:33,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/a/97ea8c415dbb4ac3b7b0bafe4a448e57 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733812113537/Put/seqid=0 2024-12-10T06:28:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741895_1073 (size=5958) 2024-12-10T06:28:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741895_1073 (size=5958) 2024-12-10T06:28:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741895_1073 (size=5958) 2024-12-10T06:28:33,629 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/a/97ea8c415dbb4ac3b7b0bafe4a448e57 2024-12-10T06:28:33,657 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/b/1ce6cdefb7424ea89701229730bb21f9 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733812113571/Put/seqid=0 2024-12-10T06:28:33,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741896_1074 (size=5958) 2024-12-10T06:28:33,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741896_1074 (size=5958) 2024-12-10T06:28:33,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741896_1074 (size=5958) 2024-12-10T06:28:33,670 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/b/1ce6cdefb7424ea89701229730bb21f9 2024-12-10T06:28:33,703 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/c/ac05e523307e400eaad4824d392a2498 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733812113580/Put/seqid=0 2024-12-10T06:28:33,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741897_1075 (size=5958) 2024-12-10T06:28:33,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741897_1075 (size=5958) 2024-12-10T06:28:33,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741897_1075 (size=5958) 2024-12-10T06:28:33,717 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/c/ac05e523307e400eaad4824d392a2498 2024-12-10T06:28:33,723 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/a/97ea8c415dbb4ac3b7b0bafe4a448e57 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/a/97ea8c415dbb4ac3b7b0bafe4a448e57 2024-12-10T06:28:33,730 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/a/97ea8c415dbb4ac3b7b0bafe4a448e57, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T06:28:33,731 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/b/1ce6cdefb7424ea89701229730bb21f9 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/b/1ce6cdefb7424ea89701229730bb21f9 2024-12-10T06:28:33,738 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/b/1ce6cdefb7424ea89701229730bb21f9, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T06:28:33,740 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/c/ac05e523307e400eaad4824d392a2498 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/c/ac05e523307e400eaad4824d392a2498 2024-12-10T06:28:33,748 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/c/ac05e523307e400eaad4824d392a2498, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T06:28:33,750 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c4214c320e558e88aac07b373bf22836 in 156ms, sequenceid=33, compaction requested=false 2024-12-10T06:28:33,750 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c4214c320e558e88aac07b373bf22836: 2024-12-10T06:28:33,750 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c4214c320e558e88aac07b373bf22836, disabling compactions & flushes 2024-12-10T06:28:33,750 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,750 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,750 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. after waiting 0 ms 2024-12-10T06:28:33,750 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,756 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. 2024-12-10T06:28:33,756 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c4214c320e558e88aac07b373bf22836: Waiting for close lock at 1733812113750Disabling compacts and flushes for region at 1733812113750Disabling writes for close at 1733812113750Writing region close event to WAL at 1733812113756 (+6 ms)Closed at 1733812113756 2024-12-10T06:28:33,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741894_1072 (size=2873) 2024-12-10T06:28:33,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741894_1072 (size=2873) 2024-12-10T06:28:33,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741894_1072 (size=2873) 2024-12-10T06:28:33,765 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/b/1ce6cdefb7424ea89701229730bb21f9 to hdfs://localhost:40333/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/b/1ce6cdefb7424ea89701229730bb21f9 2024-12-10T06:28:33,785 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494, size=2.8 K (2873bytes) 2024-12-10T06:28:33,785 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 2024-12-10T06:28:33,785 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 after 0ms 2024-12-10T06:28:33,788 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:33,788 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 took 4ms 2024-12-10T06:28:33,791 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 so closing down 2024-12-10T06:28:33,792 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:33,793 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733812113494.temp 2024-12-10T06:28:33,794 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000003-wal.1733812113494.temp 2024-12-10T06:28:33,795 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741898_1076 (size=2312) 2024-12-10T06:28:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741898_1076 (size=2312) 2024-12-10T06:28:33,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741898_1076 (size=2312) 2024-12-10T06:28:33,805 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000003-wal.1733812113494.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:33,807 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000003-wal.1733812113494.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032 2024-12-10T06:28:33,807 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-10T06:28:33,807 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494, journal: Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494, size=2.8 K (2873bytes) at 1733812113785Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 so closing down at 1733812113792 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000003-wal.1733812113494.temp at 1733812113795 (+3 ms)3 split writer threads finished at 1733812113795Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000003-wal.1733812113494.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733812113805 (+10 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000003-wal.1733812113494.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032 at 1733812113807 (+2 ms)Processed 32 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733812113807 2024-12-10T06:28:33,809 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113494 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812113494 2024-12-10T06:28:33,810 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032 2024-12-10T06:28:33,810 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:33,812 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:33,825 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113812, exclude list is [], retry=0 2024-12-10T06:28:33,828 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:33,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:33,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:33,832 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113812 2024-12-10T06:28:33,832 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:33,832 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c4214c320e558e88aac07b373bf22836, NAME => 'testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:33,833 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:33,833 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,833 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,834 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,835 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName a 2024-12-10T06:28:33,836 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,843 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/a/97ea8c415dbb4ac3b7b0bafe4a448e57 2024-12-10T06:28:33,844 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,844 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,845 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName b 2024-12-10T06:28:33,845 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,846 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,846 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,847 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4214c320e558e88aac07b373bf22836 columnFamilyName c 2024-12-10T06:28:33,847 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:33,853 DEBUG [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/c/ac05e523307e400eaad4824d392a2498 2024-12-10T06:28:33,853 INFO [StoreOpener-c4214c320e558e88aac07b373bf22836-1 {}] regionserver.HStore(327): Store=c4214c320e558e88aac07b373bf22836/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:33,853 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,854 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,856 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,856 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032 2024-12-10T06:28:33,859 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:33,860 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032 2024-12-10T06:28:33,861 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c4214c320e558e88aac07b373bf22836 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-10T06:28:33,881 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/b/473ce3a0aef24ce29d3fc57fd9cc91aa is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733812113571/Put/seqid=0 2024-12-10T06:28:33,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-10T06:28:33,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:33,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T06:28:33,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:33,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-10T06:28:33,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:33,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741900_1078 (size=5958) 2024-12-10T06:28:33,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741900_1078 (size=5958) 2024-12-10T06:28:33,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741900_1078 (size=5958) 2024-12-10T06:28:33,896 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/b/473ce3a0aef24ce29d3fc57fd9cc91aa 2024-12-10T06:28:33,902 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/.tmp/b/473ce3a0aef24ce29d3fc57fd9cc91aa as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/b/473ce3a0aef24ce29d3fc57fd9cc91aa 2024-12-10T06:28:33,908 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/b/473ce3a0aef24ce29d3fc57fd9cc91aa, entries=10, sequenceid=32, filesize=5.8 K 2024-12-10T06:28:33,909 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c4214c320e558e88aac07b373bf22836 in 48ms, sequenceid=32, compaction requested=false; wal=null 2024-12-10T06:28:33,909 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/0000000000000000032 2024-12-10T06:28:33,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,911 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,912 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:33,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4214c320e558e88aac07b373bf22836 2024-12-10T06:28:33,915 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/c4214c320e558e88aac07b373bf22836/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-10T06:28:33,916 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4214c320e558e88aac07b373bf22836; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58972651, jitterRate=-0.12123902142047882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:33,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4214c320e558e88aac07b373bf22836: Writing region info on filesystem at 1733812113833Initializing all the Stores at 1733812113834 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113834Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113834Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812113834Obtaining lock to block concurrent updates at 1733812113861 (+27 ms)Preparing flush snapshotting stores in c4214c320e558e88aac07b373bf22836 at 1733812113861Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733812113861Flushing stores of testReplayEditsWrittenViaHRegion,,1733812113399.c4214c320e558e88aac07b373bf22836. at 1733812113861Flushing c4214c320e558e88aac07b373bf22836/b: creating writer at 1733812113861Flushing c4214c320e558e88aac07b373bf22836/b: appending metadata at 1733812113881 (+20 ms)Flushing c4214c320e558e88aac07b373bf22836/b: closing flushed file at 1733812113881Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a620f4e: reopening flushed file at 1733812113901 (+20 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c4214c320e558e88aac07b373bf22836 in 48ms, sequenceid=32, compaction requested=false; wal=null at 1733812113909 (+8 ms)Cleaning up temporary data from old regions at 1733812113911 (+2 ms)Region opened successfully at 1733812113917 (+6 ms) 2024-12-10T06:28:33,941 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=415 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:51658 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:34942 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:36608 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:33844 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:48824 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1121 (was 1045) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=6338 (was 6376) 2024-12-10T06:28:33,942 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-12-10T06:28:33,957 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1121, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=6336 2024-12-10T06:28:33,957 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-12-10T06:28:33,976 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:33,979 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:33,980 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:33,982 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30135954, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-30135954, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:34,001 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30135954/hregion-30135954.1733812113983, exclude list is [], retry=0 2024-12-10T06:28:34,004 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:34,004 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:34,004 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:34,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30135954/hregion-30135954.1733812113983 2024-12-10T06:28:34,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:34,011 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d447ec455c655d4cae6c41d840f3f6ce, NAME => 'testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:34,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741902_1080 (size=68) 2024-12-10T06:28:34,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741902_1080 (size=68) 2024-12-10T06:28:34,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741902_1080 (size=68) 2024-12-10T06:28:34,422 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:34,423 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,425 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName a 2024-12-10T06:28:34,425 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:34,425 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:34,425 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,427 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName b 2024-12-10T06:28:34,427 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:34,427 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:34,427 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,428 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName c 2024-12-10T06:28:34,428 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:34,429 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:34,429 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,430 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,430 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,431 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,431 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,431 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:34,433 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,435 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:34,435 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d447ec455c655d4cae6c41d840f3f6ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69665744, jitterRate=0.038100481033325195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:34,436 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d447ec455c655d4cae6c41d840f3f6ce: Writing region info on filesystem at 1733812114422Initializing all the Stores at 1733812114423 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812114423Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812114423Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812114423Cleaning up temporary data from old regions at 1733812114431 (+8 ms)Region opened successfully at 1733812114436 (+5 ms) 2024-12-10T06:28:34,436 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d447ec455c655d4cae6c41d840f3f6ce, disabling compactions & flushes 2024-12-10T06:28:34,436 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:34,436 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:34,436 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. after waiting 0 ms 2024-12-10T06:28:34,436 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:34,436 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:34,436 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d447ec455c655d4cae6c41d840f3f6ce: Waiting for close lock at 1733812114436Disabling compacts and flushes for region at 1733812114436Disabling writes for close at 1733812114436Writing region close event to WAL at 1733812114436Closed at 1733812114436 2024-12-10T06:28:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741901_1079 (size=93) 2024-12-10T06:28:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741901_1079 (size=93) 2024-12-10T06:28:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741901_1079 (size=93) 2024-12-10T06:28:34,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:34,441 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30135954:(num 1733812113983) 2024-12-10T06:28:34,442 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:34,443 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:34,457 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444, exclude list is [], retry=0 2024-12-10T06:28:34,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:34,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:34,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:34,462 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 2024-12-10T06:28:34,462 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:34,515 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d447ec455c655d4cae6c41d840f3f6ce, NAME => 'testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:34,517 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,517 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:34,517 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,517 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,519 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,520 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName a 2024-12-10T06:28:34,520 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:34,521 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:34,521 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,522 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName b 2024-12-10T06:28:34,522 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:34,522 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:34,523 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,523 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName c 2024-12-10T06:28:34,523 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:34,524 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:34,524 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,524 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,525 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,526 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,526 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,527 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:34,528 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,529 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d447ec455c655d4cae6c41d840f3f6ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62562435, jitterRate=-0.0677470713853836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:34,529 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:34,529 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d447ec455c655d4cae6c41d840f3f6ce: Running coprocessor pre-open hook at 1733812114517Writing region info on filesystem at 1733812114517Initializing all the Stores at 1733812114519 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812114519Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812114519Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812114519Cleaning up temporary data from old regions at 1733812114526 (+7 ms)Running coprocessor post-open hooks at 1733812114529 (+3 ms)Region opened successfully at 1733812114529 2024-12-10T06:28:34,541 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d447ec455c655d4cae6c41d840f3f6ce 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-10T06:28:34,542 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:35,543 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:35,837 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:28:36,544 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:37,545 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:38,545 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:39,546 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:40,547 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:41,547 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:42,548 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:42,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T06:28:43,549 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d447ec455c655d4cae6c41d840f3f6ce/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:43,550 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d447ec455c655d4cae6c41d840f3f6ce: 2024-12-10T06:28:43,550 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:43,564 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d447ec455c655d4cae6c41d840f3f6ce: 2024-12-10T06:28:43,564 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-10T06:28:43,564 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d447ec455c655d4cae6c41d840f3f6ce, disabling compactions & flushes 2024-12-10T06:28:43,564 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:43,564 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:43,564 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. after waiting 0 ms 2024-12-10T06:28:43,564 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:43,565 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:43,565 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. 2024-12-10T06:28:43,565 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d447ec455c655d4cae6c41d840f3f6ce: Waiting for close lock at 1733812123564Running coprocessor pre-close hooks at 1733812123564Disabling compacts and flushes for region at 1733812123564Disabling writes for close at 1733812123564Writing region close event to WAL at 1733812123564Running coprocessor post-close hooks at 1733812123565 (+1 ms)Closed at 1733812123565 2024-12-10T06:28:43,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741903_1081 (size=2398) 2024-12-10T06:28:43,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741903_1081 (size=2398) 2024-12-10T06:28:43,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741903_1081 (size=2398) 2024-12-10T06:28:43,600 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444, size=2.3 K (2398bytes) 2024-12-10T06:28:43,600 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 2024-12-10T06:28:43,600 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 after 0ms 2024-12-10T06:28:43,603 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:43,603 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 took 4ms 2024-12-10T06:28:43,605 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 so closing down 2024-12-10T06:28:43,605 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:43,606 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733812114444.temp 2024-12-10T06:28:43,621 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000004-wal.1733812114444.temp 2024-12-10T06:28:43,622 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:43,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741904_1082 (size=1672) 2024-12-10T06:28:43,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741904_1082 (size=1672) 2024-12-10T06:28:43,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741904_1082 (size=1672) 2024-12-10T06:28:43,628 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000004-wal.1733812114444.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:43,630 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000004-wal.1733812114444.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026 2024-12-10T06:28:43,630 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 27 ms; skipped=3; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-10T06:28:43,630 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444, journal: Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444, size=2.3 K (2398bytes) at 1733812123600Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 so closing down at 1733812123605 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000004-wal.1733812114444.temp at 1733812123621 (+16 ms)3 split writer threads finished at 1733812123622 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000004-wal.1733812114444.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733812123629 (+7 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000004-wal.1733812114444.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026 at 1733812123630 (+1 ms)Processed 23 edits across 1 Regions in 27 ms; skipped=3; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733812123630 2024-12-10T06:28:43,632 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812114444 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812114444 2024-12-10T06:28:43,633 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026 2024-12-10T06:28:43,633 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:43,635 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:43,649 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812123635, exclude list is [], retry=0 2024-12-10T06:28:43,661 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:43,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:43,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:43,664 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812123635 2024-12-10T06:28:43,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:43,664 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d447ec455c655d4cae6c41d840f3f6ce, NAME => 'testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:43,665 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,665 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:43,665 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,665 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,667 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,668 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName a 2024-12-10T06:28:43,668 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:43,668 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:43,669 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,669 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName b 2024-12-10T06:28:43,669 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:43,670 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:43,670 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,671 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d447ec455c655d4cae6c41d840f3f6ce columnFamilyName c 2024-12-10T06:28:43,671 DEBUG [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:43,671 INFO [StoreOpener-d447ec455c655d4cae6c41d840f3f6ce-1 {}] regionserver.HStore(327): Store=d447ec455c655d4cae6c41d840f3f6ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:43,671 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,672 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,674 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026 2024-12-10T06:28:43,676 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:43,677 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026 2024-12-10T06:28:43,678 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d447ec455c655d4cae6c41d840f3f6ce 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-10T06:28:43,693 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/a/9e43bfc71ba145c39ee6b1ef9895305f is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733812123556/Put/seqid=0 2024-12-10T06:28:43,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741906_1084 (size=5523) 2024-12-10T06:28:43,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741906_1084 (size=5523) 2024-12-10T06:28:43,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741906_1084 (size=5523) 2024-12-10T06:28:43,704 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/a/9e43bfc71ba145c39ee6b1ef9895305f 2024-12-10T06:28:43,724 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/b/57fce3b8c53649b9901e5f89b75e0f28 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733812123550/Put/seqid=0 2024-12-10T06:28:43,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741907_1085 (size=5524) 2024-12-10T06:28:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741907_1085 (size=5524) 2024-12-10T06:28:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741907_1085 (size=5524) 2024-12-10T06:28:43,741 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/b/57fce3b8c53649b9901e5f89b75e0f28 2024-12-10T06:28:43,766 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/c/59b9d7b21e76462aa273f775e9723ed3 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733812123553/Put/seqid=0 2024-12-10T06:28:43,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741908_1086 (size=5457) 2024-12-10T06:28:43,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741908_1086 (size=5457) 2024-12-10T06:28:43,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741908_1086 (size=5457) 2024-12-10T06:28:43,810 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/c/59b9d7b21e76462aa273f775e9723ed3 2024-12-10T06:28:43,817 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/a/9e43bfc71ba145c39ee6b1ef9895305f as hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/a/9e43bfc71ba145c39ee6b1ef9895305f 2024-12-10T06:28:43,826 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/a/9e43bfc71ba145c39ee6b1ef9895305f, entries=7, sequenceid=26, filesize=5.4 K 2024-12-10T06:28:43,827 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/b/57fce3b8c53649b9901e5f89b75e0f28 as hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/b/57fce3b8c53649b9901e5f89b75e0f28 2024-12-10T06:28:43,833 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/b/57fce3b8c53649b9901e5f89b75e0f28, entries=7, sequenceid=26, filesize=5.4 K 2024-12-10T06:28:43,834 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/.tmp/c/59b9d7b21e76462aa273f775e9723ed3 as hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/c/59b9d7b21e76462aa273f775e9723ed3 2024-12-10T06:28:43,839 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/c/59b9d7b21e76462aa273f775e9723ed3, entries=6, sequenceid=26, filesize=5.3 K 2024-12-10T06:28:43,840 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d447ec455c655d4cae6c41d840f3f6ce in 161ms, sequenceid=26, compaction requested=false; wal=null 2024-12-10T06:28:43,840 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/0000000000000000026 2024-12-10T06:28:43,841 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,841 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,842 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:43,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,845 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsAfterAbortingFlush/d447ec455c655d4cae6c41d840f3f6ce/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-10T06:28:43,846 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d447ec455c655d4cae6c41d840f3f6ce; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59130678, jitterRate=-0.11888423562049866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:43,846 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d447ec455c655d4cae6c41d840f3f6ce 2024-12-10T06:28:43,847 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d447ec455c655d4cae6c41d840f3f6ce: Running coprocessor pre-open hook at 1733812123665Writing region info on filesystem at 1733812123665Initializing all the Stores at 1733812123666 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812123666Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812123667 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812123667Obtaining lock to block concurrent updates at 1733812123678 (+11 ms)Preparing flush snapshotting stores in d447ec455c655d4cae6c41d840f3f6ce at 1733812123678Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733812123678Flushing stores of testReplayEditsAfterAbortingFlush,,1733812113977.d447ec455c655d4cae6c41d840f3f6ce. at 1733812123678Flushing d447ec455c655d4cae6c41d840f3f6ce/a: creating writer at 1733812123678Flushing d447ec455c655d4cae6c41d840f3f6ce/a: appending metadata at 1733812123692 (+14 ms)Flushing d447ec455c655d4cae6c41d840f3f6ce/a: closing flushed file at 1733812123692Flushing d447ec455c655d4cae6c41d840f3f6ce/b: creating writer at 1733812123709 (+17 ms)Flushing d447ec455c655d4cae6c41d840f3f6ce/b: appending metadata at 1733812123723 (+14 ms)Flushing d447ec455c655d4cae6c41d840f3f6ce/b: closing flushed file at 1733812123724 (+1 ms)Flushing d447ec455c655d4cae6c41d840f3f6ce/c: creating writer at 1733812123747 (+23 ms)Flushing d447ec455c655d4cae6c41d840f3f6ce/c: appending metadata at 1733812123765 (+18 ms)Flushing d447ec455c655d4cae6c41d840f3f6ce/c: closing flushed file at 1733812123766 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1354e03e: reopening flushed file at 1733812123816 (+50 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41933a37: reopening flushed file at 1733812123826 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cf5f598: reopening flushed file at 1733812123833 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d447ec455c655d4cae6c41d840f3f6ce in 161ms, sequenceid=26, compaction requested=false; wal=null at 1733812123840 (+7 ms)Cleaning up temporary data from old regions at 1733812123841 (+1 ms)Running coprocessor post-open hooks at 1733812123846 (+5 ms)Region opened successfully at 1733812123847 (+1 ms) 2024-12-10T06:28:43,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=417 (was 415) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:56242 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:38716 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:38704 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:43722 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:43752 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:56256 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1178 (was 1121) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=328 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=6038 (was 6336) 2024-12-10T06:28:43,870 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-12-10T06:28:43,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1178, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=6038 2024-12-10T06:28:43,884 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-12-10T06:28:43,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-10T06:28:43,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:43,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-10T06:28:43,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:43,899 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:43,901 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:43,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:43,904 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37555613, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-37555613, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:43,918 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37555613/hregion-37555613.1733812123905, exclude list is [], retry=0 2024-12-10T06:28:43,938 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:43,938 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:43,938 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:43,940 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37555613/hregion-37555613.1733812123905 2024-12-10T06:28:43,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:43,941 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f20430a57b87e4dc9423e1194f8fae63, NAME => 'testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741910_1088 (size=61) 2024-12-10T06:28:43,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741910_1088 (size=61) 2024-12-10T06:28:43,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741910_1088 (size=61) 2024-12-10T06:28:43,952 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:43,953 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,954 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f20430a57b87e4dc9423e1194f8fae63 columnFamilyName a 2024-12-10T06:28:43,954 DEBUG [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:43,955 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(327): Store=f20430a57b87e4dc9423e1194f8fae63/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:43,955 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,956 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,956 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,957 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,957 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,959 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:43,983 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:43,984 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f20430a57b87e4dc9423e1194f8fae63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64101748, jitterRate=-0.04480952024459839}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:43,984 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f20430a57b87e4dc9423e1194f8fae63: Writing region info on filesystem at 1733812123952Initializing all the Stores at 1733812123952Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812123953 (+1 ms)Cleaning up temporary data from old regions at 1733812123957 (+4 ms)Region opened successfully at 1733812123984 (+27 ms) 2024-12-10T06:28:43,985 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f20430a57b87e4dc9423e1194f8fae63, disabling compactions & flushes 2024-12-10T06:28:43,985 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:43,985 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:43,985 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. after waiting 0 ms 2024-12-10T06:28:43,985 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:43,985 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:43,985 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f20430a57b87e4dc9423e1194f8fae63: Waiting for close lock at 1733812123985Disabling compacts and flushes for region at 1733812123985Disabling writes for close at 1733812123985Writing region close event to WAL at 1733812123985Closed at 1733812123985 2024-12-10T06:28:43,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741909_1087 (size=93) 2024-12-10T06:28:43,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741909_1087 (size=93) 2024-12-10T06:28:43,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741909_1087 (size=93) 2024-12-10T06:28:43,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:43,990 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37555613:(num 1733812123905) 2024-12-10T06:28:43,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:43,992 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:44,005 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992, exclude list is [], retry=0 2024-12-10T06:28:44,030 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:44,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:44,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:44,033 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 2024-12-10T06:28:44,033 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f20430a57b87e4dc9423e1194f8fae63, NAME => 'testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,035 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,036 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f20430a57b87e4dc9423e1194f8fae63 columnFamilyName a 2024-12-10T06:28:44,036 DEBUG [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,036 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(327): Store=f20430a57b87e4dc9423e1194f8fae63/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,036 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,039 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,039 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,041 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,042 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f20430a57b87e4dc9423e1194f8fae63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67970151, jitterRate=0.012834176421165466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:44,043 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f20430a57b87e4dc9423e1194f8fae63: Writing region info on filesystem at 1733812124033Initializing all the Stores at 1733812124034 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124034Cleaning up temporary data from old regions at 1733812124039 (+5 ms)Region opened successfully at 1733812124043 (+4 ms) 2024-12-10T06:28:44,059 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f20430a57b87e4dc9423e1194f8fae63, disabling compactions & flushes 2024-12-10T06:28:44,059 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:44,059 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:44,059 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. after waiting 0 ms 2024-12-10T06:28:44,059 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:44,060 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:44,060 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. 2024-12-10T06:28:44,060 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f20430a57b87e4dc9423e1194f8fae63: Waiting for close lock at 1733812124059Disabling compacts and flushes for region at 1733812124059Disabling writes for close at 1733812124059Writing region close event to WAL at 1733812124060 (+1 ms)Closed at 1733812124060 2024-12-10T06:28:44,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741911_1089 (size=838) 2024-12-10T06:28:44,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741911_1089 (size=838) 2024-12-10T06:28:44,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741911_1089 (size=838) 2024-12-10T06:28:44,091 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992, size=838 (838bytes) 2024-12-10T06:28:44,091 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 2024-12-10T06:28:44,091 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 after 0ms 2024-12-10T06:28:44,093 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:44,094 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 took 4ms 2024-12-10T06:28:44,095 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 so closing down 2024-12-10T06:28:44,095 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:44,097 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733812123992.temp 2024-12-10T06:28:44,098 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000003-wal.1733812123992.temp 2024-12-10T06:28:44,098 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741912_1090 (size=838) 2024-12-10T06:28:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741912_1090 (size=838) 2024-12-10T06:28:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741912_1090 (size=838) 2024-12-10T06:28:44,109 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000003-wal.1733812123992.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:44,110 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000003-wal.1733812123992.temp to hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012 2024-12-10T06:28:44,110 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992, size=838, length=838, corrupted=false, cancelled=false 2024-12-10T06:28:44,110 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992, journal: Splitting hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992, size=838 (838bytes) at 1733812124091Finishing writing output for hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 so closing down at 1733812124095 (+4 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000003-wal.1733812123992.temp at 1733812124098 (+3 ms)3 split writer threads finished at 1733812124098Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000003-wal.1733812123992.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733812124109 (+11 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000003-wal.1733812123992.temp to hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012 at 1733812124110 (+1 ms)Processed 10 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992, size=838, length=838, corrupted=false, cancelled=false at 1733812124110 2024-12-10T06:28:44,112 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812123992 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812123992 2024-12-10T06:28:44,113 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012 2024-12-10T06:28:44,116 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:44,433 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:44,436 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:44,457 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812124436, exclude list is [], retry=0 2024-12-10T06:28:44,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:44,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:44,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:44,462 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812124436 2024-12-10T06:28:44,464 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:44,464 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f20430a57b87e4dc9423e1194f8fae63, NAME => 'testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:44,465 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:44,465 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,465 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,468 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,470 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f20430a57b87e4dc9423e1194f8fae63 columnFamilyName a 2024-12-10T06:28:44,470 DEBUG [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,471 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(327): Store=f20430a57b87e4dc9423e1194f8fae63/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,471 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,474 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012 2024-12-10T06:28:44,476 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:44,477 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012 2024-12-10T06:28:44,477 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f20430a57b87e4dc9423e1194f8fae63 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-10T06:28:44,497 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/.tmp/a/e928785321c24bd0b39af96e40603717 is 79, key is testDatalossWhenInputError/a:x0/1733812124043/Put/seqid=0 2024-12-10T06:28:44,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741914_1092 (size=5808) 2024-12-10T06:28:44,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741914_1092 (size=5808) 2024-12-10T06:28:44,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741914_1092 (size=5808) 2024-12-10T06:28:44,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/.tmp/a/e928785321c24bd0b39af96e40603717 2024-12-10T06:28:44,515 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/.tmp/a/e928785321c24bd0b39af96e40603717 as hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/a/e928785321c24bd0b39af96e40603717 2024-12-10T06:28:44,524 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/a/e928785321c24bd0b39af96e40603717, entries=10, sequenceid=12, filesize=5.7 K 2024-12-10T06:28:44,525 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f20430a57b87e4dc9423e1194f8fae63 in 48ms, sequenceid=12, compaction requested=false; wal=null 2024-12-10T06:28:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/0000000000000000012 2024-12-10T06:28:44,527 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,527 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,530 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,533 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-10T06:28:44,534 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f20430a57b87e4dc9423e1194f8fae63; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67138142, jitterRate=4.362761974334717E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:44,534 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f20430a57b87e4dc9423e1194f8fae63: Writing region info on filesystem at 1733812124465Initializing all the Stores at 1733812124468 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124468Obtaining lock to block concurrent updates at 1733812124477 (+9 ms)Preparing flush snapshotting stores in f20430a57b87e4dc9423e1194f8fae63 at 1733812124477Finished memstore snapshotting testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733812124478 (+1 ms)Flushing stores of testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63. at 1733812124478Flushing f20430a57b87e4dc9423e1194f8fae63/a: creating writer at 1733812124478Flushing f20430a57b87e4dc9423e1194f8fae63/a: appending metadata at 1733812124496 (+18 ms)Flushing f20430a57b87e4dc9423e1194f8fae63/a: closing flushed file at 1733812124496Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63157376: reopening flushed file at 1733812124513 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f20430a57b87e4dc9423e1194f8fae63 in 48ms, sequenceid=12, compaction requested=false; wal=null at 1733812124525 (+12 ms)Cleaning up temporary data from old regions at 1733812124527 (+2 ms)Region opened successfully at 1733812124534 (+7 ms) 2024-12-10T06:28:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f20430a57b87e4dc9423e1194f8fae63, NAME => 'testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733812123899.f20430a57b87e4dc9423e1194f8fae63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,541 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,542 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f20430a57b87e4dc9423e1194f8fae63 columnFamilyName a 2024-12-10T06:28:44,542 DEBUG [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,548 DEBUG [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/a/e928785321c24bd0b39af96e40603717 2024-12-10T06:28:44,548 INFO [StoreOpener-f20430a57b87e4dc9423e1194f8fae63-1 {}] regionserver.HStore(327): Store=f20430a57b87e4dc9423e1194f8fae63/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,548 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,549 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,550 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,551 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,551 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f20430a57b87e4dc9423e1194f8fae63 2024-12-10T06:28:44,555 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testDatalossWhenInputError/f20430a57b87e4dc9423e1194f8fae63/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-10T06:28:44,557 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f20430a57b87e4dc9423e1194f8fae63; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71065064, jitterRate=0.0589519739151001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T06:28:44,557 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f20430a57b87e4dc9423e1194f8fae63: Writing region info on filesystem at 1733812124539Initializing all the Stores at 1733812124540 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124540Cleaning up temporary data from old regions at 1733812124551 (+11 ms)Region opened successfully at 1733812124557 (+6 ms) 2024-12-10T06:28:44,576 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=426 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:56344 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:38716 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:38776 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:43752 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:43822 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-616036786-172.17.0.2-1733812088637:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1178) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=328 (was 328), ProcessCount=11 (was 11), AvailableMemoryMB=6027 (was 6038) 2024-12-10T06:28:44,577 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-12-10T06:28:44,591 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=426, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=6026 2024-12-10T06:28:44,591 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-12-10T06:28:44,606 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:44,608 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:44,608 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:44,611 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35036975, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-35036975, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:44,625 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35036975/hregion-35036975.1733812124611, exclude list is [], retry=0 2024-12-10T06:28:44,628 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:44,628 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:44,628 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:44,630 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35036975/hregion-35036975.1733812124611 2024-12-10T06:28:44,631 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:44,631 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 961b7fd84b18d81e4da96a25ed5acb32, NAME => 'testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:44,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741916_1094 (size=63) 2024-12-10T06:28:44,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741916_1094 (size=63) 2024-12-10T06:28:44,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741916_1094 (size=63) 2024-12-10T06:28:44,646 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:44,651 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,653 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName a 2024-12-10T06:28:44,653 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,654 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,654 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,655 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName b 2024-12-10T06:28:44,655 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,656 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,656 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,658 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName c 2024-12-10T06:28:44,658 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,658 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,659 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,659 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,660 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,661 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,661 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,662 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:44,663 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,666 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:44,666 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 961b7fd84b18d81e4da96a25ed5acb32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74541047, jitterRate=0.11074815690517426}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:44,667 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 961b7fd84b18d81e4da96a25ed5acb32: Writing region info on filesystem at 1733812124646Initializing all the Stores at 1733812124647 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124647Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124651 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124651Cleaning up temporary data from old regions at 1733812124661 (+10 ms)Region opened successfully at 1733812124667 (+6 ms) 2024-12-10T06:28:44,667 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 961b7fd84b18d81e4da96a25ed5acb32, disabling compactions & flushes 2024-12-10T06:28:44,667 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:44,667 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:44,667 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. after waiting 0 ms 2024-12-10T06:28:44,667 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:44,668 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:44,668 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 961b7fd84b18d81e4da96a25ed5acb32: Waiting for close lock at 1733812124667Disabling compacts and flushes for region at 1733812124667Disabling writes for close at 1733812124667Writing region close event to WAL at 1733812124668 (+1 ms)Closed at 1733812124668 2024-12-10T06:28:44,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741915_1093 (size=93) 2024-12-10T06:28:44,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741915_1093 (size=93) 2024-12-10T06:28:44,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741915_1093 (size=93) 2024-12-10T06:28:44,674 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:44,674 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35036975:(num 1733812124611) 2024-12-10T06:28:44,674 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:44,676 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:44,690 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676, exclude list is [], retry=0 2024-12-10T06:28:44,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:44,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:44,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:44,695 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 2024-12-10T06:28:44,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:44,696 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 961b7fd84b18d81e4da96a25ed5acb32, NAME => 'testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:44,696 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:44,696 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,696 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,698 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,699 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName a 2024-12-10T06:28:44,699 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,699 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,699 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,700 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName b 2024-12-10T06:28:44,700 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,701 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,701 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,701 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName c 2024-12-10T06:28:44,701 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:44,702 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:44,702 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,705 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,705 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,706 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:44,707 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:44,708 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 961b7fd84b18d81e4da96a25ed5acb32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66865192, jitterRate=-0.003630995750427246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:44,709 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 961b7fd84b18d81e4da96a25ed5acb32: Writing region info on filesystem at 1733812124697Initializing all the Stores at 1733812124697Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124697Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124698 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812124698Cleaning up temporary data from old regions at 1733812124705 (+7 ms)Region opened successfully at 1733812124709 (+4 ms) 2024-12-10T06:28:44,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733812124714/Put/seqid=0 2024-12-10T06:28:44,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741918_1096 (size=4875) 2024-12-10T06:28:44,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741918_1096 (size=4875) 2024-12-10T06:28:44,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741918_1096 (size=4875) 2024-12-10T06:28:44,727 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733812124727/Put/seqid=0 2024-12-10T06:28:44,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741919_1097 (size=4875) 2024-12-10T06:28:44,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741919_1097 (size=4875) 2024-12-10T06:28:44,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741919_1097 (size=4875) 2024-12-10T06:28:44,737 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733812124737/Put/seqid=0 2024-12-10T06:28:44,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741920_1098 (size=4875) 2024-12-10T06:28:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741920_1098 (size=4875) 2024-12-10T06:28:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741920_1098 (size=4875) 2024-12-10T06:28:44,746 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 961b7fd84b18d81e4da96a25ed5acb32/a 2024-12-10T06:28:44,750 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-10T06:28:44,750 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T06:28:44,750 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 961b7fd84b18d81e4da96a25ed5acb32/a 2024-12-10T06:28:44,754 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-10T06:28:44,754 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T06:28:44,754 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 961b7fd84b18d81e4da96a25ed5acb32/a 2024-12-10T06:28:44,757 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-10T06:28:44,757 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T06:28:44,757 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 961b7fd84b18d81e4da96a25ed5acb32 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-10T06:28:44,772 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/.tmp/a/0d724352c3d2495d9067db6a8880b653 is 55, key is testCompactedBulkLoadedFiles/a:a/1733812124709/Put/seqid=0 2024-12-10T06:28:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741921_1099 (size=5107) 2024-12-10T06:28:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741921_1099 (size=5107) 2024-12-10T06:28:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741921_1099 (size=5107) 2024-12-10T06:28:44,780 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/.tmp/a/0d724352c3d2495d9067db6a8880b653 2024-12-10T06:28:44,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/.tmp/a/0d724352c3d2495d9067db6a8880b653 as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653 2024-12-10T06:28:44,790 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653, entries=1, sequenceid=4, filesize=5.0 K 2024-12-10T06:28:44,791 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 961b7fd84b18d81e4da96a25ed5acb32 in 34ms, sequenceid=4, compaction requested=false 2024-12-10T06:28:44,791 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 961b7fd84b18d81e4da96a25ed5acb32: 2024-12-10T06:28:44,792 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ 2024-12-10T06:28:44,793 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ 2024-12-10T06:28:44,794 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ 2024-12-10T06:28:44,795 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile0 into 961b7fd84b18d81e4da96a25ed5acb32/a as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ - updating store file list. 2024-12-10T06:28:44,799 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 44bd330278fd4b5db22946858b64f0b3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:44,800 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ into 961b7fd84b18d81e4da96a25ed5acb32/a 2024-12-10T06:28:44,800 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile0 into 961b7fd84b18d81e4da96a25ed5acb32/a (new location: hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_) 2024-12-10T06:28:44,800 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile1 into 961b7fd84b18d81e4da96a25ed5acb32/a as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ - updating store file list. 2024-12-10T06:28:44,805 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:44,805 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ into 961b7fd84b18d81e4da96a25ed5acb32/a 2024-12-10T06:28:44,805 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile1 into 961b7fd84b18d81e4da96a25ed5acb32/a (new location: hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_) 2024-12-10T06:28:44,806 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile2 into 961b7fd84b18d81e4da96a25ed5acb32/a as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ - updating store file list. 2024-12-10T06:28:44,810 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1a285c792f65422a980f497ebb9cc25c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:44,810 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ into 961b7fd84b18d81e4da96a25ed5acb32/a 2024-12-10T06:28:44,810 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40333/hbase/testCompactedBulkLoadedFiles/hfile2 into 961b7fd84b18d81e4da96a25ed5acb32/a (new location: hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_) 2024-12-10T06:28:44,817 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T06:28:44,817 DEBUG [Time-limited test {}] regionserver.HStore(1541): 961b7fd84b18d81e4da96a25ed5acb32/a is initiating major compaction (all files) 2024-12-10T06:28:44,818 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 961b7fd84b18d81e4da96a25ed5acb32/a in testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:44,818 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_] into tmpdir=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/.tmp, totalSize=19.3 K 2024-12-10T06:28:44,818 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0d724352c3d2495d9067db6a8880b653, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733812124709 2024-12-10T06:28:44,819 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 44bd330278fd4b5db22946858b64f0b3_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T06:28:44,819 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T06:28:44,819 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1a285c792f65422a980f497ebb9cc25c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T06:28:44,831 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/.tmp/a/32d3ae3290d44ffc9baff64373365478 is 55, key is testCompactedBulkLoadedFiles/a:a/1733812124709/Put/seqid=0 2024-12-10T06:28:44,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741922_1100 (size=6154) 2024-12-10T06:28:44,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741922_1100 (size=6154) 2024-12-10T06:28:44,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741922_1100 (size=6154) 2024-12-10T06:28:44,848 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/.tmp/a/32d3ae3290d44ffc9baff64373365478 as hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/32d3ae3290d44ffc9baff64373365478 2024-12-10T06:28:44,855 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 961b7fd84b18d81e4da96a25ed5acb32/a of 961b7fd84b18d81e4da96a25ed5acb32 into 32d3ae3290d44ffc9baff64373365478(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T06:28:44,855 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 961b7fd84b18d81e4da96a25ed5acb32: 2024-12-10T06:28:44,855 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-10T06:28:44,855 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-10T06:28:44,887 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676, size=0 (0bytes) 2024-12-10T06:28:44,887 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 might be still open, length is 0 2024-12-10T06:28:44,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 2024-12-10T06:28:44,888 WARN [IPC Server handler 0 on default port 40333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-10T06:28:44,888 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 after 1ms 2024-12-10T06:28:45,513 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:38806 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38806 dst: /127.0.0.1:40347 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40347 remote=/127.0.0.1:38806]. Total timeout mills is 60000, 59342 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:45,513 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:56370 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56370 dst: /127.0.0.1:45191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:45,513 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:43872 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33683:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43872 dst: /127.0.0.1:33683 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:45,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741917_1101 (size=1368) 2024-12-10T06:28:45,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741917_1101 (size=1368) 2024-12-10T06:28:48,889 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 after 4002ms 2024-12-10T06:28:48,892 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:48,892 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 took 4005ms 2024-12-10T06:28:48,894 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676; continuing. 2024-12-10T06:28:48,894 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 so closing down 2024-12-10T06:28:48,894 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:48,895 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733812124676.temp 2024-12-10T06:28:48,896 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000003-wal.1733812124676.temp 2024-12-10T06:28:48,896 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741923_1102 (size=635) 2024-12-10T06:28:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741923_1102 (size=635) 2024-12-10T06:28:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741923_1102 (size=635) 2024-12-10T06:28:48,903 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000003-wal.1733812124676.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:48,904 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000003-wal.1733812124676.temp to hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008 2024-12-10T06:28:48,904 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T06:28:48,904 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676, journal: Splitting hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676, size=0 (0bytes) at 1733812124887Finishing writing output for hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 so closing down at 1733812128894 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000003-wal.1733812124676.temp at 1733812128896 (+2 ms)3 split writer threads finished at 1733812128896Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000003-wal.1733812124676.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733812128903 (+7 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000003-wal.1733812124676.temp to hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008 at 1733812128904 (+1 ms)Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676, size=0, length=0, corrupted=false, cancelled=false at 1733812128904 2024-12-10T06:28:48,905 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812124676 2024-12-10T06:28:48,906 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008 2024-12-10T06:28:48,906 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:48,908 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:48,921 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812128908, exclude list is [], retry=0 2024-12-10T06:28:48,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:48,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:48,924 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:48,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812128908 2024-12-10T06:28:48,925 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:48,926 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 961b7fd84b18d81e4da96a25ed5acb32, NAME => 'testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:48,926 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:48,926 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,926 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,927 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,928 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName a 2024-12-10T06:28:48,928 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:48,933 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653 2024-12-10T06:28:48,936 DEBUG [StoreFileOpener-961b7fd84b18d81e4da96a25ed5acb32-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1a285c792f65422a980f497ebb9cc25c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:48,936 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ 2024-12-10T06:28:48,940 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/32d3ae3290d44ffc9baff64373365478 2024-12-10T06:28:48,944 DEBUG [StoreFileOpener-961b7fd84b18d81e4da96a25ed5acb32-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 44bd330278fd4b5db22946858b64f0b3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:48,944 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ 2024-12-10T06:28:48,947 DEBUG [StoreFileOpener-961b7fd84b18d81e4da96a25ed5acb32-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T06:28:48,947 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ 2024-12-10T06:28:48,947 WARN [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@55ba53fb 2024-12-10T06:28:48,947 WARN [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@55ba53fb 2024-12-10T06:28:48,947 WARN [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@55ba53fb 2024-12-10T06:28:48,947 WARN [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@55ba53fb 2024-12-10T06:28:48,947 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_] to archive 2024-12-10T06:28:48,948 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T06:28:48,950 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653 to hdfs://localhost:40333/hbase/archive/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/0d724352c3d2495d9067db6a8880b653 2024-12-10T06:28:48,950 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ to hdfs://localhost:40333/hbase/archive/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/1a285c792f65422a980f497ebb9cc25c_SeqId_4_ 2024-12-10T06:28:48,950 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ to hdfs://localhost:40333/hbase/archive/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/44bd330278fd4b5db22946858b64f0b3_SeqId_4_ 2024-12-10T06:28:48,950 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ to hdfs://localhost:40333/hbase/archive/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/a/59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_ 2024-12-10T06:28:48,950 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:48,951 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,951 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName b 2024-12-10T06:28:48,951 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:48,952 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:48,952 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,952 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 961b7fd84b18d81e4da96a25ed5acb32 columnFamilyName c 2024-12-10T06:28:48,953 DEBUG [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:48,953 INFO [StoreOpener-961b7fd84b18d81e4da96a25ed5acb32-1 {}] regionserver.HStore(327): Store=961b7fd84b18d81e4da96a25ed5acb32/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:48,953 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,955 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,956 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008 2024-12-10T06:28:48,957 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:48,959 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 961b7fd84b18d81e4da96a25ed5acb32 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "961b7fd84b18d81e4da96a25ed5acb32" family_name: "a" compaction_input: "0d724352c3d2495d9067db6a8880b653" compaction_input: "44bd330278fd4b5db22946858b64f0b3_SeqId_4_" compaction_input: "59b2fa0012d24e78bdc5f5f44c0e7cbc_SeqId_4_" compaction_input: "1a285c792f65422a980f497ebb9cc25c_SeqId_4_" compaction_output: "32d3ae3290d44ffc9baff64373365478" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-10T06:28:48,959 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-10T06:28:48,959 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008 2024-12-10T06:28:48,960 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/0000000000000000008 2024-12-10T06:28:48,961 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,961 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,961 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:48,963 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 961b7fd84b18d81e4da96a25ed5acb32 2024-12-10T06:28:48,965 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testCompactedBulkLoadedFiles/961b7fd84b18d81e4da96a25ed5acb32/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-10T06:28:48,967 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 961b7fd84b18d81e4da96a25ed5acb32; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74959912, jitterRate=0.11698973178863525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:48,967 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 961b7fd84b18d81e4da96a25ed5acb32: Writing region info on filesystem at 1733812128926Initializing all the Stores at 1733812128927 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812128927Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812128927Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812128927Cleaning up temporary data from old regions at 1733812128961 (+34 ms)Region opened successfully at 1733812128967 (+6 ms) 2024-12-10T06:28:48,969 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 961b7fd84b18d81e4da96a25ed5acb32, disabling compactions & flushes 2024-12-10T06:28:48,969 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:48,969 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:48,969 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. after waiting 0 ms 2024-12-10T06:28:48,969 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:48,970 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733812124606.961b7fd84b18d81e4da96a25ed5acb32. 2024-12-10T06:28:48,970 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 961b7fd84b18d81e4da96a25ed5acb32: Waiting for close lock at 1733812128969Disabling compacts and flushes for region at 1733812128969Disabling writes for close at 1733812128969Writing region close event to WAL at 1733812128970 (+1 ms)Closed at 1733812128970 2024-12-10T06:28:48,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741924_1103 (size=93) 2024-12-10T06:28:48,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741924_1103 (size=93) 2024-12-10T06:28:48,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741924_1103 (size=93) 2024-12-10T06:28:48,975 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:48,975 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733812128908) 2024-12-10T06:28:48,991 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=443 (was 426) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-76376197_22 at /127.0.0.1:43918 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-76376197_22 at /127.0.0.1:56436 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40333 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1343 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 328) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5962 (was 6026) 2024-12-10T06:28:48,991 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1343 is superior to 1024 2024-12-10T06:28:49,010 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1343, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=5959 2024-12-10T06:28:49,011 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1343 is superior to 1024 2024-12-10T06:28:49,027 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:49,028 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T06:28:49,029 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T06:28:49,031 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-52089378, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/hregion-52089378, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:49,045 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-52089378/hregion-52089378.1733812129031, exclude list is [], retry=0 2024-12-10T06:28:49,048 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:49,048 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:49,048 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:49,050 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-52089378/hregion-52089378.1733812129031 2024-12-10T06:28:49,050 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175)] 2024-12-10T06:28:49,050 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0f110a38ddb2709610c8749173f4be2f, NAME => 'testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40333/hbase 2024-12-10T06:28:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741926_1105 (size=67) 2024-12-10T06:28:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741926_1105 (size=67) 2024-12-10T06:28:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741926_1105 (size=67) 2024-12-10T06:28:49,061 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:49,062 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,063 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName a 2024-12-10T06:28:49,063 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,064 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,064 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,065 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName b 2024-12-10T06:28:49,065 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,066 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,066 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,067 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName c 2024-12-10T06:28:49,067 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,068 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,068 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,069 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,069 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,070 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:49,071 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,073 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T06:28:49,073 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0f110a38ddb2709610c8749173f4be2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66306252, jitterRate=-0.011959850788116455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0f110a38ddb2709610c8749173f4be2f: Writing region info on filesystem at 1733812129061Initializing all the Stores at 1733812129062 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129062Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129062Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129062Cleaning up temporary data from old regions at 1733812129069 (+7 ms)Region opened successfully at 1733812129073 (+4 ms) 2024-12-10T06:28:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0f110a38ddb2709610c8749173f4be2f, disabling compactions & flushes 2024-12-10T06:28:49,073 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. after waiting 0 ms 2024-12-10T06:28:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,074 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,074 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0f110a38ddb2709610c8749173f4be2f: Waiting for close lock at 1733812129073Disabling compacts and flushes for region at 1733812129073Disabling writes for close at 1733812129073Writing region close event to WAL at 1733812129074 (+1 ms)Closed at 1733812129074 2024-12-10T06:28:49,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741925_1104 (size=93) 2024-12-10T06:28:49,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741925_1104 (size=93) 2024-12-10T06:28:49,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741925_1104 (size=93) 2024-12-10T06:28:49,076 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-52089378/hregion-52089378.1733812129031 not finished, retry = 0 2024-12-10T06:28:49,086 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T06:28:49,179 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:49,179 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-52089378:(num 1733812129031) 2024-12-10T06:28:49,179 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:49,181 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:49,195 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181, exclude list is [], retry=0 2024-12-10T06:28:49,197 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:49,198 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:49,198 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:49,199 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 2024-12-10T06:28:49,200 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f110a38ddb2709610c8749173f4be2f, NAME => 'testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,202 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,203 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName a 2024-12-10T06:28:49,203 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,204 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,204 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,204 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName b 2024-12-10T06:28:49,204 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,205 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,205 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,205 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName c 2024-12-10T06:28:49,206 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,206 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,206 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,208 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,208 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,208 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,209 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:49,210 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,211 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0f110a38ddb2709610c8749173f4be2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72359262, jitterRate=0.07823702692985535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:49,211 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0f110a38ddb2709610c8749173f4be2f: Writing region info on filesystem at 1733812129200Initializing all the Stores at 1733812129201 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129201Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129202 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129202Cleaning up temporary data from old regions at 1733812129208 (+6 ms)Region opened successfully at 1733812129211 (+3 ms) 2024-12-10T06:28:49,218 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0f110a38ddb2709610c8749173f4be2f 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-10T06:28:49,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/a/e65fb03c5a3e472d86d573ddd83cb661 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733812129211/Put/seqid=0 2024-12-10T06:28:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741928_1107 (size=5958) 2024-12-10T06:28:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741928_1107 (size=5958) 2024-12-10T06:28:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741928_1107 (size=5958) 2024-12-10T06:28:49,242 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/a/e65fb03c5a3e472d86d573ddd83cb661 2024-12-10T06:28:49,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/a/e65fb03c5a3e472d86d573ddd83cb661 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/a/e65fb03c5a3e472d86d573ddd83cb661 2024-12-10T06:28:49,253 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/a/e65fb03c5a3e472d86d573ddd83cb661, entries=10, sequenceid=13, filesize=5.8 K 2024-12-10T06:28:49,254 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 0f110a38ddb2709610c8749173f4be2f in 36ms, sequenceid=13, compaction requested=false 2024-12-10T06:28:49,254 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0f110a38ddb2709610c8749173f4be2f: 2024-12-10T06:28:49,272 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0f110a38ddb2709610c8749173f4be2f, disabling compactions & flushes 2024-12-10T06:28:49,272 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,272 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,272 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. after waiting 0 ms 2024-12-10T06:28:49,272 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,277 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,278 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:49,278 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0f110a38ddb2709610c8749173f4be2f: Waiting for close lock at 1733812129272Disabling compacts and flushes for region at 1733812129272Disabling writes for close at 1733812129272Writing region close event to WAL at 1733812129277 (+5 ms)Closed at 1733812129278 (+1 ms) 2024-12-10T06:28:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741927_1106 (size=2805) 2024-12-10T06:28:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741927_1106 (size=2805) 2024-12-10T06:28:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741927_1106 (size=2805) 2024-12-10T06:28:49,305 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181, size=2.7 K (2805bytes) 2024-12-10T06:28:49,305 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 2024-12-10T06:28:49,305 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 after 0ms 2024-12-10T06:28:49,307 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:49,308 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 took 4ms 2024-12-10T06:28:49,309 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 so closing down 2024-12-10T06:28:49,309 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:49,310 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733812129181.temp 2024-12-10T06:28:49,311 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000003-wal.1733812129181.temp 2024-12-10T06:28:49,312 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741929_1108 (size=2312) 2024-12-10T06:28:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741929_1108 (size=2312) 2024-12-10T06:28:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741929_1108 (size=2312) 2024-12-10T06:28:49,318 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000003-wal.1733812129181.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:49,319 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000003-wal.1733812129181.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035 2024-12-10T06:28:49,319 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-10T06:28:49,319 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181, journal: Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181, size=2.7 K (2805bytes) at 1733812129305Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 so closing down at 1733812129309 (+4 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000003-wal.1733812129181.temp at 1733812129312 (+3 ms)3 split writer threads finished at 1733812129312Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000003-wal.1733812129181.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733812129318 (+6 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000003-wal.1733812129181.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035 at 1733812129319 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733812129319 2024-12-10T06:28:49,321 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129181 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812129181 2024-12-10T06:28:49,322 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035 2024-12-10T06:28:49,322 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:49,323 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:49,336 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323, exclude list is [], retry=0 2024-12-10T06:28:49,338 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:49,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:49,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:49,340 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 2024-12-10T06:28:49,340 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33133:33133),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:40515:40515)] 2024-12-10T06:28:49,341 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f110a38ddb2709610c8749173f4be2f, NAME => 'testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T06:28:49,341 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:49,341 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,341 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,342 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,343 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName a 2024-12-10T06:28:49,343 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,348 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/a/e65fb03c5a3e472d86d573ddd83cb661 2024-12-10T06:28:49,348 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,348 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,349 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName b 2024-12-10T06:28:49,349 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,349 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,349 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,350 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName c 2024-12-10T06:28:49,350 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:49,350 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:49,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,351 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,354 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035 2024-12-10T06:28:49,355 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:49,356 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035 2024-12-10T06:28:49,356 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0f110a38ddb2709610c8749173f4be2f 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-10T06:28:49,371 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/b/7f277a19db0e475cb03d0101fc232ffe is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733812129254/Put/seqid=0 2024-12-10T06:28:49,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741931_1110 (size=5958) 2024-12-10T06:28:49,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741931_1110 (size=5958) 2024-12-10T06:28:49,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741931_1110 (size=5958) 2024-12-10T06:28:49,378 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/b/7f277a19db0e475cb03d0101fc232ffe 2024-12-10T06:28:49,396 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/c/e1f10da4f30b4a54a5c7f053403579ee is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733812129260/Put/seqid=0 2024-12-10T06:28:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741932_1111 (size=5958) 2024-12-10T06:28:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741932_1111 (size=5958) 2024-12-10T06:28:49,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741932_1111 (size=5958) 2024-12-10T06:28:49,403 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/c/e1f10da4f30b4a54a5c7f053403579ee 2024-12-10T06:28:49,408 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/b/7f277a19db0e475cb03d0101fc232ffe as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/b/7f277a19db0e475cb03d0101fc232ffe 2024-12-10T06:28:49,412 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/b/7f277a19db0e475cb03d0101fc232ffe, entries=10, sequenceid=35, filesize=5.8 K 2024-12-10T06:28:49,413 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/c/e1f10da4f30b4a54a5c7f053403579ee as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/c/e1f10da4f30b4a54a5c7f053403579ee 2024-12-10T06:28:49,417 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/c/e1f10da4f30b4a54a5c7f053403579ee, entries=10, sequenceid=35, filesize=5.8 K 2024-12-10T06:28:49,418 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 0f110a38ddb2709610c8749173f4be2f in 62ms, sequenceid=35, compaction requested=false; wal=null 2024-12-10T06:28:49,418 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000035 2024-12-10T06:28:49,419 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,419 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,420 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:49,421 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:49,423 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-10T06:28:49,423 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0f110a38ddb2709610c8749173f4be2f; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72684604, jitterRate=0.08308500051498413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:49,424 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0f110a38ddb2709610c8749173f4be2f: Writing region info on filesystem at 1733812129341Initializing all the Stores at 1733812129342 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129342Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129342Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812129342Obtaining lock to block concurrent updates at 1733812129357 (+15 ms)Preparing flush snapshotting stores in 0f110a38ddb2709610c8749173f4be2f at 1733812129357Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733812129357Flushing stores of testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. at 1733812129357Flushing 0f110a38ddb2709610c8749173f4be2f/b: creating writer at 1733812129357Flushing 0f110a38ddb2709610c8749173f4be2f/b: appending metadata at 1733812129370 (+13 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/b: closing flushed file at 1733812129370Flushing 0f110a38ddb2709610c8749173f4be2f/c: creating writer at 1733812129382 (+12 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/c: appending metadata at 1733812129395 (+13 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/c: closing flushed file at 1733812129395Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49bb8737: reopening flushed file at 1733812129407 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@616fa9cc: reopening flushed file at 1733812129413 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 0f110a38ddb2709610c8749173f4be2f in 62ms, sequenceid=35, compaction requested=false; wal=null at 1733812129418 (+5 ms)Cleaning up temporary data from old regions at 1733812129419 (+1 ms)Region opened successfully at 1733812129424 (+5 ms) 2024-12-10T06:28:49,485 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323, size=0 (0bytes) 2024-12-10T06:28:49,485 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 might be still open, length is 0 2024-12-10T06:28:49,485 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 2024-12-10T06:28:49,486 WARN [IPC Server handler 0 on default port 40333 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-10T06:28:49,486 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 after 1ms 2024-12-10T06:28:51,512 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:56526 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56526 dst: /127.0.0.1:45191 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45191 remote=/127.0.0.1:56526]. Total timeout mills is 60000, 57937 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:51,512 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:38920 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38920 dst: /127.0.0.1:40347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:51,512 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-756945437_22 at /127.0.0.1:43954 [Receiving block BP-616036786-172.17.0.2-1733812088637:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33683:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43954 dst: /127.0.0.1:33683 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741930_1112 (size=2304) 2024-12-10T06:28:51,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741930_1112 (size=2304) 2024-12-10T06:28:53,487 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 after 4002ms 2024-12-10T06:28:53,490 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:53,490 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 took 4005ms 2024-12-10T06:28:53,492 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323; continuing. 2024-12-10T06:28:53,492 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 so closing down 2024-12-10T06:28:53,492 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T06:28:53,493 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733812129323.temp 2024-12-10T06:28:53,494 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000037-wal.1733812129323.temp 2024-12-10T06:28:53,494 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T06:28:53,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741933_1113 (size=2312) 2024-12-10T06:28:53,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741933_1113 (size=2312) 2024-12-10T06:28:53,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741933_1113 (size=2312) 2024-12-10T06:28:53,501 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000037-wal.1733812129323.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T06:28:53,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000037-wal.1733812129323.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066 2024-12-10T06:28:53,502 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T06:28:53,502 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323, journal: Splitting hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323, size=0 (0bytes) at 1733812129485Finishing writing output for hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 so closing down at 1733812133492 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000037-wal.1733812129323.temp at 1733812133494 (+2 ms)3 split writer threads finished at 1733812133495 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000037-wal.1733812129323.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733812133501 (+6 ms)Rename recovered edits hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000037-wal.1733812129323.temp to hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066 at 1733812133502 (+1 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323, size=0, length=0, corrupted=false, cancelled=false at 1733812133502 2024-12-10T06:28:53,504 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 to hdfs://localhost:40333/hbase/oldWALs/wal.1733812129323 2024-12-10T06:28:53,505 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066 2024-12-10T06:28:53,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T06:28:53,506 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40333/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026, archiveDir=hdfs://localhost:40333/hbase/oldWALs, maxLogs=32 2024-12-10T06:28:53,520 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812133507, exclude list is [], retry=0 2024-12-10T06:28:53,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33683,DS-f17827d8-43e9-4c4d-a644-e003bd771ad2,DISK] 2024-12-10T06:28:53,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40347,DS-3840f433-b6cc-44df-9edf-8679ca4e88a0,DISK] 2024-12-10T06:28:53,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45191,DS-12b57370-5005-4de0-ba73-054d878d9d64,DISK] 2024-12-10T06:28:53,524 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812133507 2024-12-10T06:28:53,524 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40515:40515),(127.0.0.1/127.0.0.1:33175:33175),(127.0.0.1/127.0.0.1:33133:33133)] 2024-12-10T06:28:53,525 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T06:28:53,526 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,526 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName a 2024-12-10T06:28:53,526 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:53,531 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/a/e65fb03c5a3e472d86d573ddd83cb661 2024-12-10T06:28:53,531 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:53,531 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,532 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName b 2024-12-10T06:28:53,532 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:53,536 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/b/7f277a19db0e475cb03d0101fc232ffe 2024-12-10T06:28:53,536 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:53,536 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,537 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f110a38ddb2709610c8749173f4be2f columnFamilyName c 2024-12-10T06:28:53,537 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T06:28:53,541 DEBUG [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/c/e1f10da4f30b4a54a5c7f053403579ee 2024-12-10T06:28:53,541 INFO [StoreOpener-0f110a38ddb2709610c8749173f4be2f-1 {}] regionserver.HStore(327): Store=0f110a38ddb2709610c8749173f4be2f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T06:28:53,541 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,542 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,543 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,544 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066 2024-12-10T06:28:53,545 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T06:28:53,550 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066 2024-12-10T06:28:53,550 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0f110a38ddb2709610c8749173f4be2f 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-10T06:28:53,565 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/a/415540ad9df64e3caffb34855106767d is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733812129429/Put/seqid=0 2024-12-10T06:28:53,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741935_1115 (size=5958) 2024-12-10T06:28:53,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741935_1115 (size=5958) 2024-12-10T06:28:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741935_1115 (size=5958) 2024-12-10T06:28:53,571 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/a/415540ad9df64e3caffb34855106767d 2024-12-10T06:28:53,589 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/b/c15e505f565c4f8d837839c8bce0e567 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733812129436/Put/seqid=0 2024-12-10T06:28:53,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741936_1116 (size=5958) 2024-12-10T06:28:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741936_1116 (size=5958) 2024-12-10T06:28:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741936_1116 (size=5958) 2024-12-10T06:28:53,596 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/b/c15e505f565c4f8d837839c8bce0e567 2024-12-10T06:28:53,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/c/1119e677ae9e46ca9579a36ab89b714f is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733812129442/Put/seqid=0 2024-12-10T06:28:53,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741937_1117 (size=5958) 2024-12-10T06:28:53,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741937_1117 (size=5958) 2024-12-10T06:28:53,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741937_1117 (size=5958) 2024-12-10T06:28:53,648 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/c/1119e677ae9e46ca9579a36ab89b714f 2024-12-10T06:28:53,654 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/a/415540ad9df64e3caffb34855106767d as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/a/415540ad9df64e3caffb34855106767d 2024-12-10T06:28:53,658 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/a/415540ad9df64e3caffb34855106767d, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T06:28:53,659 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/b/c15e505f565c4f8d837839c8bce0e567 as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/b/c15e505f565c4f8d837839c8bce0e567 2024-12-10T06:28:53,663 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/b/c15e505f565c4f8d837839c8bce0e567, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T06:28:53,664 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/.tmp/c/1119e677ae9e46ca9579a36ab89b714f as hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/c/1119e677ae9e46ca9579a36ab89b714f 2024-12-10T06:28:53,668 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/c/1119e677ae9e46ca9579a36ab89b714f, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T06:28:53,668 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 0f110a38ddb2709610c8749173f4be2f in 118ms, sequenceid=66, compaction requested=false; wal=null 2024-12-10T06:28:53,669 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/0000000000000000066 2024-12-10T06:28:53,670 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,670 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,670 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T06:28:53,672 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0f110a38ddb2709610c8749173f4be2f 2024-12-10T06:28:53,674 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/hbase/data/default/testReplayEditsWrittenViaHRegion/0f110a38ddb2709610c8749173f4be2f/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-10T06:28:53,674 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0f110a38ddb2709610c8749173f4be2f; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62845676, jitterRate=-0.063526451587677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T06:28:53,675 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0f110a38ddb2709610c8749173f4be2f: Writing region info on filesystem at 1733812133525Initializing all the Stores at 1733812133525Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812133525Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812133526 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733812133526Obtaining lock to block concurrent updates at 1733812133550 (+24 ms)Preparing flush snapshotting stores in 0f110a38ddb2709610c8749173f4be2f at 1733812133550Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733812133550Flushing stores of testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. at 1733812133550Flushing 0f110a38ddb2709610c8749173f4be2f/a: creating writer at 1733812133550Flushing 0f110a38ddb2709610c8749173f4be2f/a: appending metadata at 1733812133564 (+14 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/a: closing flushed file at 1733812133564Flushing 0f110a38ddb2709610c8749173f4be2f/b: creating writer at 1733812133576 (+12 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/b: appending metadata at 1733812133589 (+13 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/b: closing flushed file at 1733812133589Flushing 0f110a38ddb2709610c8749173f4be2f/c: creating writer at 1733812133601 (+12 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/c: appending metadata at 1733812133614 (+13 ms)Flushing 0f110a38ddb2709610c8749173f4be2f/c: closing flushed file at 1733812133614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47473f99: reopening flushed file at 1733812133653 (+39 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37846551: reopening flushed file at 1733812133658 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f9299b7: reopening flushed file at 1733812133663 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 0f110a38ddb2709610c8749173f4be2f in 118ms, sequenceid=66, compaction requested=false; wal=null at 1733812133668 (+5 ms)Cleaning up temporary data from old regions at 1733812133670 (+2 ms)Region opened successfully at 1733812133675 (+5 ms) 2024-12-10T06:28:53,688 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0f110a38ddb2709610c8749173f4be2f, disabling compactions & flushes 2024-12-10T06:28:53,688 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:53,688 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:53,688 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. after waiting 0 ms 2024-12-10T06:28:53,688 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:53,690 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733812129027.0f110a38ddb2709610c8749173f4be2f. 2024-12-10T06:28:53,690 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0f110a38ddb2709610c8749173f4be2f: Waiting for close lock at 1733812133688Disabling compacts and flushes for region at 1733812133688Disabling writes for close at 1733812133688Writing region close event to WAL at 1733812133690 (+2 ms)Closed at 1733812133690 2024-12-10T06:28:53,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741934_1114 (size=93) 2024-12-10T06:28:53,693 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812133507 not finished, retry = 0 2024-12-10T06:28:53,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741934_1114 (size=93) 2024-12-10T06:28:53,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741934_1114 (size=93) 2024-12-10T06:28:53,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T06:28:53,796 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733812133507) 2024-12-10T06:28:53,814 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=450 (was 443) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40333 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1756084725_22 at /127.0.0.1:52584 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1756084725_22 at /127.0.0.1:34396 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40333 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1756084725_22 at /127.0.0.1:54362 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1343) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 334), ProcessCount=11 (was 11), AvailableMemoryMB=5884 (was 5959) 2024-12-10T06:28:53,814 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-12-10T06:28:53,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T06:28:53,815 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:28:53,815 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:28:53,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:53,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:53,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T06:28:53,816 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T06:28:53,816 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=234563806, stopped=false 2024-12-10T06:28:53,816 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c42c682b34f7,34871,1733812093474 2024-12-10T06:28:53,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:53,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:53,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T06:28:53,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:53,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:53,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:53,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T06:28:53,827 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T06:28:53,827 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:28:53,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:53,828 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c42c682b34f7,41301,1733812094511' ***** 2024-12-10T06:28:53,828 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T06:28:53,828 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c42c682b34f7,35565,1733812094566' ***** 2024-12-10T06:28:53,828 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T06:28:53,828 INFO [RS:1;c42c682b34f7:41301 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T06:28:53,828 INFO [RS:1;c42c682b34f7:41301 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T06:28:53,828 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:53,828 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:53,828 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:53,828 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T06:28:53,829 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(959): stopping server c42c682b34f7,41301,1733812094511 2024-12-10T06:28:53,829 INFO [RS:1;c42c682b34f7:41301 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T06:28:53,829 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T06:28:53,829 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a37328b6903c9afef567df7be16e039, disabling compactions & flushes 2024-12-10T06:28:53,829 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T06:28:53,829 INFO [RS:1;c42c682b34f7:41301 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c42c682b34f7:41301. 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(959): stopping server c42c682b34f7,35565,1733812094566 2024-12-10T06:28:53,829 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T06:28:53,829 DEBUG [RS:1;c42c682b34f7:41301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:28:53,829 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:53,829 DEBUG [RS:1;c42c682b34f7:41301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c42c682b34f7:35565. 2024-12-10T06:28:53,829 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. after waiting 0 ms 2024-12-10T06:28:53,829 DEBUG [RS:2;c42c682b34f7:35565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T06:28:53,829 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:53,829 DEBUG [RS:2;c42c682b34f7:35565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:53,829 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T06:28:53,829 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T06:28:53,829 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T06:28:53,829 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(976): stopping server c42c682b34f7,35565,1733812094566; all regions closed. 2024-12-10T06:28:53,830 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T06:28:53,830 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T06:28:53,830 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T06:28:53,830 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1325): Online Regions={8a37328b6903c9afef567df7be16e039=testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T06:28:53,830 DEBUG [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8a37328b6903c9afef567df7be16e039 2024-12-10T06:28:53,830 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T06:28:53,830 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T06:28:53,830 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T06:28:53,830 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T06:28:53,831 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T06:28:53,831 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-10T06:28:53,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741833_1009 (size=93) 2024-12-10T06:28:53,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741833_1009 (size=93) 2024-12-10T06:28:53,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741833_1009 (size=93) 2024-12-10T06:28:53,835 DEBUG [RS:2;c42c682b34f7:35565 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs 2024-12-10T06:28:53,835 INFO [RS:2;c42c682b34f7:35565 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c42c682b34f7%2C35565%2C1733812094566:(num 1733812096198) 2024-12-10T06:28:53,835 DEBUG [RS:2;c42c682b34f7:35565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:53,835 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T06:28:53,836 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a37328b6903c9afef567df7be16e039/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] hbase.ChoreService(370): Chore service for: regionserver/c42c682b34f7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T06:28:53,836 INFO [regionserver/c42c682b34f7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T06:28:53,836 INFO [RS:2;c42c682b34f7:35565 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35565 2024-12-10T06:28:53,837 INFO [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:53,837 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a37328b6903c9afef567df7be16e039: Waiting for close lock at 1733812133829Running coprocessor pre-close hooks at 1733812133829Disabling compacts and flushes for region at 1733812133829Disabling writes for close at 1733812133829Writing region close event to WAL at 1733812133831 (+2 ms)Running coprocessor post-close hooks at 1733812133836 (+5 ms)Closed at 1733812133837 (+1 ms) 2024-12-10T06:28:53,837 DEBUG [RS_CLOSE_REGION-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039. 2024-12-10T06:28:53,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c42c682b34f7,35565,1733812094566 2024-12-10T06:28:53,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T06:28:53,850 INFO [RS:2;c42c682b34f7:35565 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T06:28:53,861 INFO [regionserver/c42c682b34f7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:28:53,861 INFO [regionserver/c42c682b34f7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:28:53,862 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/info/3270c468617b48628a91d885a77f4c4f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733812109872.8a37328b6903c9afef567df7be16e039./info:regioninfo/1733812113055/Put/seqid=0 2024-12-10T06:28:53,865 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c42c682b34f7,35565,1733812094566] 2024-12-10T06:28:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741938_1118 (size=8243) 2024-12-10T06:28:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741938_1118 (size=8243) 2024-12-10T06:28:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741938_1118 (size=8243) 2024-12-10T06:28:53,869 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/info/3270c468617b48628a91d885a77f4c4f 2024-12-10T06:28:53,880 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c42c682b34f7,35565,1733812094566 already deleted, retry=false 2024-12-10T06:28:53,881 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c42c682b34f7,35565,1733812094566 expired; onlineServers=1 2024-12-10T06:28:53,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-10T06:28:53,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:53,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-10T06:28:53,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-10T06:28:53,889 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/ns/f9d66e6141d44376a56ef80f56cc0725 is 43, key is default/ns:d/1733812096917/Put/seqid=0 2024-12-10T06:28:53,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T06:28:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741939_1119 (size=5153) 2024-12-10T06:28:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741939_1119 (size=5153) 2024-12-10T06:28:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741939_1119 (size=5153) 2024-12-10T06:28:53,896 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/ns/f9d66e6141d44376a56ef80f56cc0725 2024-12-10T06:28:53,915 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/table/022e1f390eaf4d2fba0eaa3da1704b60 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733812110360/Put/seqid=0 2024-12-10T06:28:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741940_1120 (size=5431) 2024-12-10T06:28:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741940_1120 (size=5431) 2024-12-10T06:28:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741940_1120 (size=5431) 2024-12-10T06:28:53,921 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/table/022e1f390eaf4d2fba0eaa3da1704b60 2024-12-10T06:28:53,926 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/info/3270c468617b48628a91d885a77f4c4f as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/info/3270c468617b48628a91d885a77f4c4f 2024-12-10T06:28:53,931 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/info/3270c468617b48628a91d885a77f4c4f, entries=18, sequenceid=21, filesize=8.0 K 2024-12-10T06:28:53,932 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/ns/f9d66e6141d44376a56ef80f56cc0725 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/ns/f9d66e6141d44376a56ef80f56cc0725 2024-12-10T06:28:53,937 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/ns/f9d66e6141d44376a56ef80f56cc0725, entries=2, sequenceid=21, filesize=5.0 K 2024-12-10T06:28:53,937 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/.tmp/table/022e1f390eaf4d2fba0eaa3da1704b60 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/table/022e1f390eaf4d2fba0eaa3da1704b60 2024-12-10T06:28:53,942 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/table/022e1f390eaf4d2fba0eaa3da1704b60, entries=2, sequenceid=21, filesize=5.3 K 2024-12-10T06:28:53,943 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=21, compaction requested=false 2024-12-10T06:28:53,947 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-10T06:28:53,948 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T06:28:53,948 INFO [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T06:28:53,948 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733812133830Running coprocessor pre-close hooks at 1733812133830Disabling compacts and flushes for region at 1733812133830Disabling writes for close at 1733812133830Obtaining lock to block concurrent updates at 1733812133831 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733812133831Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733812133831Flushing stores of hbase:meta,,1.1588230740 at 1733812133833 (+2 ms)Flushing 1588230740/info: creating writer at 1733812133833Flushing 1588230740/info: appending metadata at 1733812133861 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733812133861Flushing 1588230740/ns: creating writer at 1733812133875 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733812133889 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733812133889Flushing 1588230740/table: creating writer at 1733812133901 (+12 ms)Flushing 1588230740/table: appending metadata at 1733812133914 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733812133914Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e4a53bc: reopening flushed file at 1733812133926 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65916167: reopening flushed file at 1733812133932 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bf6529b: reopening flushed file at 1733812133937 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=21, compaction requested=false at 1733812133943 (+6 ms)Writing region close event to WAL at 1733812133945 (+2 ms)Running coprocessor post-close hooks at 1733812133948 (+3 ms)Closed at 1733812133948 2024-12-10T06:28:53,948 DEBUG [RS_CLOSE_META-regionserver/c42c682b34f7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T06:28:53,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:53,965 INFO [RS:2;c42c682b34f7:35565 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T06:28:53,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35565-0x1019b0ee98c0003, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:53,965 INFO [RS:2;c42c682b34f7:35565 {}] regionserver.HRegionServer(1031): Exiting; stopping=c42c682b34f7,35565,1733812094566; zookeeper connection closed. 2024-12-10T06:28:53,966 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@598dde08 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@598dde08 2024-12-10T06:28:54,030 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(976): stopping server c42c682b34f7,41301,1733812094511; all regions closed. 2024-12-10T06:28:54,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741836_1012 (size=3783) 2024-12-10T06:28:54,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741836_1012 (size=3783) 2024-12-10T06:28:54,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741836_1012 (size=3783) 2024-12-10T06:28:54,036 DEBUG [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs 2024-12-10T06:28:54,036 INFO [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c42c682b34f7%2C41301%2C1733812094511.meta:.meta(num 1733812096736) 2024-12-10T06:28:54,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741835_1011 (size=2652) 2024-12-10T06:28:54,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741835_1011 (size=2652) 2024-12-10T06:28:54,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741835_1011 (size=2652) 2024-12-10T06:28:54,040 DEBUG [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/oldWALs 2024-12-10T06:28:54,040 INFO [RS:1;c42c682b34f7:41301 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c42c682b34f7%2C41301%2C1733812094511:(num 1733812096198) 2024-12-10T06:28:54,040 DEBUG [RS:1;c42c682b34f7:41301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T06:28:54,040 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T06:28:54,041 INFO [RS:1;c42c682b34f7:41301 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T06:28:54,041 INFO [RS:1;c42c682b34f7:41301 {}] hbase.ChoreService(370): Chore service for: regionserver/c42c682b34f7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T06:28:54,041 INFO [RS:1;c42c682b34f7:41301 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T06:28:54,041 INFO [regionserver/c42c682b34f7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T06:28:54,041 INFO [RS:1;c42c682b34f7:41301 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41301 2024-12-10T06:28:54,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c42c682b34f7,41301,1733812094511 2024-12-10T06:28:54,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T06:28:54,057 INFO [RS:1;c42c682b34f7:41301 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T06:28:54,073 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c42c682b34f7,41301,1733812094511] 2024-12-10T06:28:54,088 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c42c682b34f7,41301,1733812094511 already deleted, retry=false 2024-12-10T06:28:54,088 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c42c682b34f7,41301,1733812094511 expired; onlineServers=0 2024-12-10T06:28:54,088 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c42c682b34f7,34871,1733812093474' ***** 2024-12-10T06:28:54,088 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T06:28:54,088 INFO [M:0;c42c682b34f7:34871 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T06:28:54,088 INFO [M:0;c42c682b34f7:34871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T06:28:54,088 DEBUG [M:0;c42c682b34f7:34871 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T06:28:54,089 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T06:28:54,089 DEBUG [M:0;c42c682b34f7:34871 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T06:28:54,089 DEBUG [master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.small.0-1733812095825 {}] cleaner.HFileCleaner(306): Exit Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.small.0-1733812095825,5,FailOnTimeoutGroup] 2024-12-10T06:28:54,089 DEBUG [master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.large.0-1733812095820 {}] cleaner.HFileCleaner(306): Exit Thread[master/c42c682b34f7:0:becomeActiveMaster-HFileCleaner.large.0-1733812095820,5,FailOnTimeoutGroup] 2024-12-10T06:28:54,089 INFO [M:0;c42c682b34f7:34871 {}] hbase.ChoreService(370): Chore service for: master/c42c682b34f7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T06:28:54,089 INFO [M:0;c42c682b34f7:34871 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T06:28:54,089 DEBUG [M:0;c42c682b34f7:34871 {}] master.HMaster(1795): Stopping service threads 2024-12-10T06:28:54,089 INFO [M:0;c42c682b34f7:34871 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T06:28:54,089 INFO [M:0;c42c682b34f7:34871 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T06:28:54,090 INFO [M:0;c42c682b34f7:34871 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T06:28:54,090 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T06:28:54,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T06:28:54,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T06:28:54,104 DEBUG [M:0;c42c682b34f7:34871 {}] zookeeper.ZKUtil(347): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T06:28:54,105 WARN [M:0;c42c682b34f7:34871 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T06:28:54,105 INFO [M:0;c42c682b34f7:34871 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/.lastflushedseqids 2024-12-10T06:28:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741941_1121 (size=138) 2024-12-10T06:28:54,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741941_1121 (size=138) 2024-12-10T06:28:54,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741941_1121 (size=138) 2024-12-10T06:28:54,118 INFO [M:0;c42c682b34f7:34871 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T06:28:54,119 INFO [M:0;c42c682b34f7:34871 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T06:28:54,119 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T06:28:54,119 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:54,119 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:54,119 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T06:28:54,119 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:54,119 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-10T06:28:54,134 DEBUG [M:0;c42c682b34f7:34871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7e5ed8570e5848d09efbcc3d86c66fb8 is 82, key is hbase:meta,,1/info:regioninfo/1733812096832/Put/seqid=0 2024-12-10T06:28:54,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741942_1122 (size=5672) 2024-12-10T06:28:54,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741942_1122 (size=5672) 2024-12-10T06:28:54,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741942_1122 (size=5672) 2024-12-10T06:28:54,142 INFO [M:0;c42c682b34f7:34871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7e5ed8570e5848d09efbcc3d86c66fb8 2024-12-10T06:28:54,161 DEBUG [M:0;c42c682b34f7:34871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8de837602dd4a31af7ae2e8660005d9 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733812110367/Put/seqid=0 2024-12-10T06:28:54,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741943_1123 (size=7755) 2024-12-10T06:28:54,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741943_1123 (size=7755) 2024-12-10T06:28:54,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741943_1123 (size=7755) 2024-12-10T06:28:54,168 INFO [M:0;c42c682b34f7:34871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8de837602dd4a31af7ae2e8660005d9 2024-12-10T06:28:54,172 INFO [M:0;c42c682b34f7:34871 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e8de837602dd4a31af7ae2e8660005d9 2024-12-10T06:28:54,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:54,173 INFO [RS:1;c42c682b34f7:41301 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T06:28:54,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41301-0x1019b0ee98c0002, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:54,173 INFO [RS:1;c42c682b34f7:41301 {}] regionserver.HRegionServer(1031): Exiting; stopping=c42c682b34f7,41301,1733812094511; zookeeper connection closed. 2024-12-10T06:28:54,173 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b89d165 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b89d165 2024-12-10T06:28:54,174 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T06:28:54,187 DEBUG [M:0;c42c682b34f7:34871 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/233e4e03358a447080c5f1d690a32acd is 69, key is c42c682b34f7,35565,1733812094566/rs:state/1733812095913/Put/seqid=0 2024-12-10T06:28:54,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741944_1124 (size=5440) 2024-12-10T06:28:54,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741944_1124 (size=5440) 2024-12-10T06:28:54,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741944_1124 (size=5440) 2024-12-10T06:28:54,194 INFO [M:0;c42c682b34f7:34871 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/233e4e03358a447080c5f1d690a32acd 2024-12-10T06:28:54,199 INFO [M:0;c42c682b34f7:34871 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 233e4e03358a447080c5f1d690a32acd 2024-12-10T06:28:54,199 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7e5ed8570e5848d09efbcc3d86c66fb8 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7e5ed8570e5848d09efbcc3d86c66fb8 2024-12-10T06:28:54,204 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7e5ed8570e5848d09efbcc3d86c66fb8, entries=8, sequenceid=168, filesize=5.5 K 2024-12-10T06:28:54,205 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8de837602dd4a31af7ae2e8660005d9 as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8de837602dd4a31af7ae2e8660005d9 2024-12-10T06:28:54,211 INFO [M:0;c42c682b34f7:34871 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e8de837602dd4a31af7ae2e8660005d9 2024-12-10T06:28:54,211 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8de837602dd4a31af7ae2e8660005d9, entries=17, sequenceid=168, filesize=7.6 K 2024-12-10T06:28:54,212 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/233e4e03358a447080c5f1d690a32acd as hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/233e4e03358a447080c5f1d690a32acd 2024-12-10T06:28:54,217 INFO [M:0;c42c682b34f7:34871 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 233e4e03358a447080c5f1d690a32acd 2024-12-10T06:28:54,217 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40333/user/jenkins/test-data/e1c92e53-e76d-bfe4-e7a9-0559eba6fb25/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/233e4e03358a447080c5f1d690a32acd, entries=3, sequenceid=168, filesize=5.3 K 2024-12-10T06:28:54,218 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=168, compaction requested=false 2024-12-10T06:28:54,219 INFO [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T06:28:54,219 DEBUG [M:0;c42c682b34f7:34871 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733812134119Disabling compacts and flushes for region at 1733812134119Disabling writes for close at 1733812134119Obtaining lock to block concurrent updates at 1733812134119Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733812134119Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733812134119Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733812134120 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733812134120Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733812134134 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733812134134Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733812134146 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733812134161 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733812134161Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733812134172 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733812134186 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733812134186Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360c1bcb: reopening flushed file at 1733812134199 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38c7b7bc: reopening flushed file at 1733812134204 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@678b150f: reopening flushed file at 1733812134211 (+7 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=168, compaction requested=false at 1733812134218 (+7 ms)Writing region close event to WAL at 1733812134219 (+1 ms)Closed at 1733812134219 2024-12-10T06:28:54,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40347 is added to blk_1073741830_1006 (size=71337) 2024-12-10T06:28:54,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741830_1006 (size=71337) 2024-12-10T06:28:54,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33683 is added to blk_1073741830_1006 (size=71337) 2024-12-10T06:28:54,223 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T06:28:54,223 INFO [M:0;c42c682b34f7:34871 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T06:28:54,223 INFO [M:0;c42c682b34f7:34871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34871 2024-12-10T06:28:54,224 INFO [M:0;c42c682b34f7:34871 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T06:28:54,334 INFO [M:0;c42c682b34f7:34871 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T06:28:54,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:54,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34871-0x1019b0ee98c0000, quorum=127.0.0.1:59695, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T06:28:54,341 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113812 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113812 (inode 16655) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733812113398/wal.1733812113812 (inode 16655) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T06:28:54,341 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733812105376/wal.1733812105477 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:54,342 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733812129026/wal.1733812129323 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:54,344 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812123635 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812123635 (inode 16678) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733812113976/wal.1733812123635 (inode 16678) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T06:28:54,344 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733812098776/wal.1733812098848 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:54,346 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812124436 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812124436 (inode 16704) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733812123898/wal.1733812124436 (inode 16704) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T06:28:54,347 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733812124605/wal.1733812124676 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T06:28:54,349 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal.1733812098176 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal.1733812098176 (inode 16485) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733812097789/wal.1733812098176 (inode 16485) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T06:28:54,351 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal.1733812097474 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal.1733812097474 (inode 16462) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733812097176/wal.1733812097474 (inode 16462) Holder DFSClient_NONMAPREDUCE_-756945437_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T06:28:54,355 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:28:54,357 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T06:28:54,357 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T06:28:54,358 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T06:28:54,358 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,STOPPED} 2024-12-10T06:28:54,360 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T06:28:54,360 WARN [BP-616036786-172.17.0.2-1733812088637 heartbeating to localhost/127.0.0.1:40333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T06:28:54,360 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T06:28:54,360 WARN [BP-616036786-172.17.0.2-1733812088637 heartbeating to localhost/127.0.0.1:40333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-616036786-172.17.0.2-1733812088637 (Datanode Uuid ebf1f6bc-fd40-4f24-9c9d-8bf323c1bc33) service to localhost/127.0.0.1:40333 2024-12-10T06:28:54,362 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data5/current/BP-616036786-172.17.0.2-1733812088637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:28:54,362 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data6/current/BP-616036786-172.17.0.2-1733812088637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:28:54,362 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T06:28:54,364 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:28:54,364 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T06:28:54,364 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T06:28:54,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T06:28:54,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,STOPPED} 2024-12-10T06:28:54,366 WARN [BP-616036786-172.17.0.2-1733812088637 heartbeating to localhost/127.0.0.1:40333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T06:28:54,366 WARN [BP-616036786-172.17.0.2-1733812088637 heartbeating to localhost/127.0.0.1:40333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-616036786-172.17.0.2-1733812088637 (Datanode Uuid 3c5c76b2-33e8-4b82-af04-f232a7b8eaa7) service to localhost/127.0.0.1:40333 2024-12-10T06:28:54,366 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T06:28:54,366 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T06:28:54,366 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data3/current/BP-616036786-172.17.0.2-1733812088637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:28:54,367 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data4/current/BP-616036786-172.17.0.2-1733812088637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:28:54,367 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T06:28:54,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T06:28:54,374 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T06:28:54,374 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T06:28:54,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T06:28:54,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,STOPPED} 2024-12-10T06:28:54,375 WARN [BP-616036786-172.17.0.2-1733812088637 heartbeating to localhost/127.0.0.1:40333 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T06:28:54,375 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T06:28:54,375 WARN [BP-616036786-172.17.0.2-1733812088637 heartbeating to localhost/127.0.0.1:40333 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-616036786-172.17.0.2-1733812088637 (Datanode Uuid e03296b0-875f-446a-93af-aef12208d63e) service to localhost/127.0.0.1:40333 2024-12-10T06:28:54,375 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T06:28:54,376 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data1/current/BP-616036786-172.17.0.2-1733812088637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:28:54,376 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/cluster_c172fe78-3867-f6d6-b99f-cc19aeb4b294/data/data2/current/BP-616036786-172.17.0.2-1733812088637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T06:28:54,376 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T06:28:54,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T06:28:54,383 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T06:28:54,383 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T06:28:54,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T06:28:54,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/af0bf59e-b5bb-e1e3-4d21-8f69b75fbc54/hadoop.log.dir/,STOPPED} 2024-12-10T06:28:54,393 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T06:28:54,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down