2024-11-13 08:28:12,241 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 08:28:12,252 main DEBUG Took 0.008798 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 08:28:12,252 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 08:28:12,252 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 08:28:12,253 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 08:28:12,254 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,261 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 08:28:12,273 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,276 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,277 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,278 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,279 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,281 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,283 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,284 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,285 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,286 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,287 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 08:28:12,288 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,288 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 08:28:12,290 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 08:28:12,291 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 08:28:12,293 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 08:28:12,294 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 08:28:12,295 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 08:28:12,296 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 08:28:12,307 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 08:28:12,311 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 08:28:12,313 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 08:28:12,314 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 08:28:12,314 main DEBUG createAppenders(={Console}) 2024-11-13 08:28:12,315 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-13 08:28:12,315 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 08:28:12,316 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-13 08:28:12,317 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 08:28:12,317 main DEBUG OutputStream closed 2024-11-13 08:28:12,317 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 08:28:12,318 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 08:28:12,318 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-13 08:28:12,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 08:28:12,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 08:28:12,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 08:28:12,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 08:28:12,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 08:28:12,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 08:28:12,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 08:28:12,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 08:28:12,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 08:28:12,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 08:28:12,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 08:28:12,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 08:28:12,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 08:28:12,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 08:28:12,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 08:28:12,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 08:28:12,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 08:28:12,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 08:28:12,409 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 08:28:12,409 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-13 08:28:12,410 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 08:28:12,410 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-13T08:28:12,669 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f 2024-11-13 08:28:12,672 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 08:28:12,672 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T08:28:12,682 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T08:28:12,707 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T08:28:12,759 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T08:28:12,759 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T08:28:12,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T08:28:12,786 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd, deleteOnExit=true 2024-11-13T08:28:12,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T08:28:12,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/test.cache.data in system properties and HBase conf 2024-11-13T08:28:12,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T08:28:12,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir in system properties and HBase conf 2024-11-13T08:28:12,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T08:28:12,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T08:28:12,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T08:28:12,875 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T08:28:12,977 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T08:28:12,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T08:28:12,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T08:28:12,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T08:28:12,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T08:28:12,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T08:28:12,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T08:28:12,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T08:28:12,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T08:28:12,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T08:28:12,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/nfs.dump.dir in system properties and HBase conf 2024-11-13T08:28:12,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/java.io.tmpdir in system properties and HBase conf 2024-11-13T08:28:12,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T08:28:12,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T08:28:12,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T08:28:13,809 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T08:28:13,885 INFO [Time-limited test {}] log.Log(170): Logging initialized @2298ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T08:28:13,959 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T08:28:14,018 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T08:28:14,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T08:28:14,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T08:28:14,039 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T08:28:14,054 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T08:28:14,058 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,AVAILABLE} 2024-11-13T08:28:14,059 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T08:28:14,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/java.io.tmpdir/jetty-localhost-40973-hadoop-hdfs-3_4_1-tests_jar-_-any-13638162038552307982/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T08:28:14,265 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:40973} 2024-11-13T08:28:14,266 INFO [Time-limited test {}] server.Server(415): Started @2679ms 2024-11-13T08:28:14,641 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T08:28:14,648 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T08:28:14,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T08:28:14,653 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T08:28:14,654 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T08:28:14,654 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,AVAILABLE} 2024-11-13T08:28:14,655 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T08:28:14,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/java.io.tmpdir/jetty-localhost-38037-hadoop-hdfs-3_4_1-tests_jar-_-any-16637014595646202172/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T08:28:14,788 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:38037} 2024-11-13T08:28:14,788 INFO [Time-limited test {}] server.Server(415): Started @3201ms 2024-11-13T08:28:14,844 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T08:28:14,973 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T08:28:14,980 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T08:28:14,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T08:28:14,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T08:28:14,983 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T08:28:14,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,AVAILABLE} 2024-11-13T08:28:14,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T08:28:15,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/java.io.tmpdir/jetty-localhost-34113-hadoop-hdfs-3_4_1-tests_jar-_-any-8605645462745591885/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T08:28:15,128 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:34113} 2024-11-13T08:28:15,128 INFO [Time-limited test {}] server.Server(415): Started @3541ms 2024-11-13T08:28:15,130 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T08:28:15,185 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T08:28:15,192 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T08:28:15,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T08:28:15,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T08:28:15,199 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T08:28:15,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,AVAILABLE} 2024-11-13T08:28:15,200 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T08:28:15,312 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data2/current/BP-433341908-172.17.0.2-1731486493558/current, will proceed with Du for space computation calculation, 2024-11-13T08:28:15,312 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data1/current/BP-433341908-172.17.0.2-1731486493558/current, will proceed with Du for space computation calculation, 2024-11-13T08:28:15,313 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data4/current/BP-433341908-172.17.0.2-1731486493558/current, will proceed with Du for space computation calculation, 2024-11-13T08:28:15,312 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data3/current/BP-433341908-172.17.0.2-1731486493558/current, will proceed with Du for space computation calculation, 2024-11-13T08:28:15,330 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/java.io.tmpdir/jetty-localhost-36241-hadoop-hdfs-3_4_1-tests_jar-_-any-4987135586735120345/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T08:28:15,332 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:36241} 2024-11-13T08:28:15,333 INFO [Time-limited test {}] server.Server(415): Started @3746ms 2024-11-13T08:28:15,335 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T08:28:15,358 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T08:28:15,358 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T08:28:15,439 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb642e7c69f9f1236 with lease ID 0x51e047d397cda2ae: Processing first storage report for DS-1b8f8a47-8ede-4400-a165-80b0a456379b from datanode DatanodeRegistration(127.0.0.1:34115, datanodeUuid=2e7281ce-1c99-4aae-81ad-031d5d06ac21, infoPort=37895, infoSecurePort=0, ipcPort=33907, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558) 2024-11-13T08:28:15,441 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb642e7c69f9f1236 with lease ID 0x51e047d397cda2ae: from storage DS-1b8f8a47-8ede-4400-a165-80b0a456379b node DatanodeRegistration(127.0.0.1:34115, datanodeUuid=2e7281ce-1c99-4aae-81ad-031d5d06ac21, infoPort=37895, infoSecurePort=0, ipcPort=33907, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T08:28:15,441 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf16375bf49b00cff with lease ID 0x51e047d397cda2ad: Processing first storage report for DS-78f58666-22d4-4130-8364-a840009c3df9 from datanode DatanodeRegistration(127.0.0.1:44221, datanodeUuid=14c44a06-c44b-494c-a5eb-ac0c550a074b, infoPort=39765, infoSecurePort=0, ipcPort=37607, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558) 2024-11-13T08:28:15,441 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf16375bf49b00cff with lease ID 0x51e047d397cda2ad: from storage DS-78f58666-22d4-4130-8364-a840009c3df9 node DatanodeRegistration(127.0.0.1:44221, datanodeUuid=14c44a06-c44b-494c-a5eb-ac0c550a074b, infoPort=39765, infoSecurePort=0, ipcPort=37607, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T08:28:15,442 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb642e7c69f9f1236 with lease ID 0x51e047d397cda2ae: Processing first storage report for DS-3b45a520-004d-4883-9244-6005597df360 from datanode DatanodeRegistration(127.0.0.1:34115, datanodeUuid=2e7281ce-1c99-4aae-81ad-031d5d06ac21, infoPort=37895, infoSecurePort=0, ipcPort=33907, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558) 2024-11-13T08:28:15,442 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb642e7c69f9f1236 with lease ID 0x51e047d397cda2ae: from storage DS-3b45a520-004d-4883-9244-6005597df360 node DatanodeRegistration(127.0.0.1:34115, datanodeUuid=2e7281ce-1c99-4aae-81ad-031d5d06ac21, infoPort=37895, infoSecurePort=0, ipcPort=33907, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T08:28:15,444 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf16375bf49b00cff with lease ID 0x51e047d397cda2ad: Processing first storage report for DS-9f4f6890-89a2-4482-886b-8e5067ed0cb2 from datanode DatanodeRegistration(127.0.0.1:44221, datanodeUuid=14c44a06-c44b-494c-a5eb-ac0c550a074b, infoPort=39765, infoSecurePort=0, ipcPort=37607, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558) 2024-11-13T08:28:15,445 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf16375bf49b00cff with lease ID 0x51e047d397cda2ad: from storage DS-9f4f6890-89a2-4482-886b-8e5067ed0cb2 node DatanodeRegistration(127.0.0.1:44221, datanodeUuid=14c44a06-c44b-494c-a5eb-ac0c550a074b, infoPort=39765, infoSecurePort=0, ipcPort=37607, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T08:28:15,473 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data5/current/BP-433341908-172.17.0.2-1731486493558/current, will proceed with Du for space computation calculation, 2024-11-13T08:28:15,474 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data6/current/BP-433341908-172.17.0.2-1731486493558/current, will proceed with Du for space computation calculation, 2024-11-13T08:28:15,512 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T08:28:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1701b09b072c214e with lease ID 0x51e047d397cda2af: Processing first storage report for DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4 from datanode DatanodeRegistration(127.0.0.1:34057, datanodeUuid=f421a173-ef64-4f0d-bddb-2955923dedc1, infoPort=41159, infoSecurePort=0, ipcPort=35013, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558) 2024-11-13T08:28:15,523 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1701b09b072c214e with lease ID 0x51e047d397cda2af: from storage DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4 node DatanodeRegistration(127.0.0.1:34057, datanodeUuid=f421a173-ef64-4f0d-bddb-2955923dedc1, infoPort=41159, infoSecurePort=0, ipcPort=35013, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T08:28:15,523 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1701b09b072c214e with lease ID 0x51e047d397cda2af: Processing first storage report for DS-f962e632-ff9a-44ef-bb57-3c5a7c345add from datanode DatanodeRegistration(127.0.0.1:34057, datanodeUuid=f421a173-ef64-4f0d-bddb-2955923dedc1, infoPort=41159, infoSecurePort=0, ipcPort=35013, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558) 2024-11-13T08:28:15,523 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1701b09b072c214e with lease ID 0x51e047d397cda2af: from storage DS-f962e632-ff9a-44ef-bb57-3c5a7c345add node DatanodeRegistration(127.0.0.1:34057, datanodeUuid=f421a173-ef64-4f0d-bddb-2955923dedc1, infoPort=41159, infoSecurePort=0, ipcPort=35013, storageInfo=lv=-57;cid=testClusterID;nsid=94160632;c=1731486493558), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T08:28:15,740 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f 2024-11-13T08:28:15,817 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/zookeeper_0, clientPort=62074, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T08:28:15,829 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62074 2024-11-13T08:28:15,843 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:15,846 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:16,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741825_1001 (size=7) 2024-11-13T08:28:16,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741825_1001 (size=7) 2024-11-13T08:28:16,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741825_1001 (size=7) 2024-11-13T08:28:16,489 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc with version=8 2024-11-13T08:28:16,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/hbase-staging 2024-11-13T08:28:16,792 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dd3a0865bb23:0 server-side Connection retries=45 2024-11-13T08:28:16,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:16,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:16,808 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T08:28:16,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:16,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T08:28:16,947 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T08:28:17,009 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T08:28:17,019 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T08:28:17,023 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T08:28:17,052 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18960 (auto-detected) 2024-11-13T08:28:17,053 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T08:28:17,073 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37177 2024-11-13T08:28:17,094 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37177 connecting to ZooKeeper ensemble=127.0.0.1:62074 2024-11-13T08:28:17,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:371770x0, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T08:28:17,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37177-0x10110711ec40000 connected 2024-11-13T08:28:17,163 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,166 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,179 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:17,184 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc, hbase.cluster.distributed=false 2024-11-13T08:28:17,210 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T08:28:17,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37177 2024-11-13T08:28:17,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37177 2024-11-13T08:28:17,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37177 2024-11-13T08:28:17,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37177 2024-11-13T08:28:17,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37177 2024-11-13T08:28:17,332 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd3a0865bb23:0 server-side Connection retries=45 2024-11-13T08:28:17,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,334 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T08:28:17,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T08:28:17,338 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T08:28:17,340 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T08:28:17,341 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35727 2024-11-13T08:28:17,343 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35727 connecting to ZooKeeper ensemble=127.0.0.1:62074 2024-11-13T08:28:17,344 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,347 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:357270x0, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T08:28:17,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:357270x0, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:17,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35727-0x10110711ec40001 connected 2024-11-13T08:28:17,358 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T08:28:17,366 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T08:28:17,368 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T08:28:17,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T08:28:17,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35727 2024-11-13T08:28:17,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35727 2024-11-13T08:28:17,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35727 2024-11-13T08:28:17,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35727 2024-11-13T08:28:17,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35727 2024-11-13T08:28:17,392 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd3a0865bb23:0 server-side Connection retries=45 2024-11-13T08:28:17,392 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,393 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T08:28:17,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T08:28:17,393 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T08:28:17,394 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T08:28:17,394 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44073 2024-11-13T08:28:17,396 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44073 connecting to ZooKeeper ensemble=127.0.0.1:62074 2024-11-13T08:28:17,397 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440730x0, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T08:28:17,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44073-0x10110711ec40002 connected 2024-11-13T08:28:17,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:17,405 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T08:28:17,406 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T08:28:17,407 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T08:28:17,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T08:28:17,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44073 2024-11-13T08:28:17,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44073 2024-11-13T08:28:17,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44073 2024-11-13T08:28:17,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44073 2024-11-13T08:28:17,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44073 2024-11-13T08:28:17,433 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd3a0865bb23:0 server-side Connection retries=45 2024-11-13T08:28:17,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,433 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T08:28:17,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T08:28:17,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T08:28:17,434 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T08:28:17,434 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T08:28:17,435 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45659 2024-11-13T08:28:17,437 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45659 connecting to ZooKeeper ensemble=127.0.0.1:62074 2024-11-13T08:28:17,438 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,440 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456590x0, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T08:28:17,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456590x0, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:17,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45659-0x10110711ec40003 connected 2024-11-13T08:28:17,448 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T08:28:17,449 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T08:28:17,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T08:28:17,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T08:28:17,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45659 2024-11-13T08:28:17,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45659 2024-11-13T08:28:17,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45659 2024-11-13T08:28:17,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45659 2024-11-13T08:28:17,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45659 2024-11-13T08:28:17,475 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd3a0865bb23:37177 2024-11-13T08:28:17,476 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,489 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,510 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T08:28:17,512 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd3a0865bb23,37177,1731486496596 from backup master directory 2024-11-13T08:28:17,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:17,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T08:28:17,516 WARN [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T08:28:17,516 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:17,519 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T08:28:17,520 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T08:28:17,578 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/hbase.id] with ID: 976bee63-cb40-472b-8f8e-0f050f8e2d48 2024-11-13T08:28:17,578 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/.tmp/hbase.id 2024-11-13T08:28:17,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741826_1002 (size=42) 2024-11-13T08:28:17,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741826_1002 (size=42) 2024-11-13T08:28:17,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741826_1002 (size=42) 2024-11-13T08:28:17,593 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/.tmp/hbase.id]:[hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/hbase.id] 2024-11-13T08:28:17,635 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T08:28:17,639 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T08:28:17,658 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-13T08:28:17,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:17,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741827_1003 (size=196) 2024-11-13T08:28:17,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741827_1003 (size=196) 2024-11-13T08:28:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741827_1003 (size=196) 2024-11-13T08:28:17,696 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T08:28:17,698 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T08:28:17,704 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:17,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741828_1004 (size=1189) 2024-11-13T08:28:17,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741828_1004 (size=1189) 2024-11-13T08:28:17,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741828_1004 (size=1189) 2024-11-13T08:28:17,759 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store 2024-11-13T08:28:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741829_1005 (size=34) 2024-11-13T08:28:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741829_1005 (size=34) 2024-11-13T08:28:17,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741829_1005 (size=34) 2024-11-13T08:28:17,788 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T08:28:17,790 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:17,792 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T08:28:17,792 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:17,792 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:17,793 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T08:28:17,794 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:17,794 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:17,795 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731486497791Disabling compacts and flushes for region at 1731486497791Disabling writes for close at 1731486497793 (+2 ms)Writing region close event to WAL at 1731486497794 (+1 ms)Closed at 1731486497794 2024-11-13T08:28:17,796 WARN [master/dd3a0865bb23:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/.initializing 2024-11-13T08:28:17,796 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/WALs/dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:17,804 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:17,818 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd3a0865bb23%2C37177%2C1731486496596, suffix=, logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/WALs/dd3a0865bb23,37177,1731486496596, archiveDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/oldWALs, maxLogs=10 2024-11-13T08:28:17,847 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/WALs/dd3a0865bb23,37177,1731486496596/dd3a0865bb23%2C37177%2C1731486496596.1731486497823, exclude list is [], retry=0 2024-11-13T08:28:17,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:17,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:17,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:17,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:17,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T08:28:17,912 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/WALs/dd3a0865bb23,37177,1731486496596/dd3a0865bb23%2C37177%2C1731486496596.1731486497823 2024-11-13T08:28:17,913 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:17,914 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:17,914 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:17,917 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:17,918 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:17,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:17,980 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T08:28:17,984 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:17,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:17,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:17,990 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T08:28:17,990 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:17,991 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:17,991 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:17,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T08:28:17,993 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:17,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:17,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:17,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T08:28:17,996 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:17,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:17,998 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:18,000 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:18,001 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:18,006 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:18,007 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:18,010 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T08:28:18,013 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T08:28:18,017 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:18,019 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70355968, jitterRate=0.0483856201171875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T08:28:18,024 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731486497930Initializing all the Stores at 1731486497932 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486497933 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486497933Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486497933Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486497933Cleaning up temporary data from old regions at 1731486498007 (+74 ms)Region opened successfully at 1731486498024 (+17 ms) 2024-11-13T08:28:18,025 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T08:28:18,060 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72763ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd3a0865bb23/172.17.0.2:0 2024-11-13T08:28:18,093 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T08:28:18,104 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T08:28:18,104 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T08:28:18,107 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T08:28:18,109 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T08:28:18,114 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T08:28:18,114 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T08:28:18,140 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T08:28:18,148 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T08:28:18,152 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T08:28:18,154 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T08:28:18,156 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T08:28:18,158 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T08:28:18,160 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T08:28:18,163 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T08:28:18,165 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T08:28:18,166 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T08:28:18,168 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T08:28:18,186 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T08:28:18,188 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T08:28:18,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:18,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:18,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:18,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:18,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,196 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd3a0865bb23,37177,1731486496596, sessionid=0x10110711ec40000, setting cluster-up flag (Was=false) 2024-11-13T08:28:18,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,219 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T08:28:18,221 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:18,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,233 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T08:28:18,234 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:18,240 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T08:28:18,264 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(746): ClusterId : 976bee63-cb40-472b-8f8e-0f050f8e2d48 2024-11-13T08:28:18,264 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(746): ClusterId : 976bee63-cb40-472b-8f8e-0f050f8e2d48 2024-11-13T08:28:18,264 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(746): ClusterId : 976bee63-cb40-472b-8f8e-0f050f8e2d48 2024-11-13T08:28:18,267 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T08:28:18,267 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T08:28:18,267 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T08:28:18,273 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T08:28:18,273 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T08:28:18,273 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T08:28:18,273 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T08:28:18,274 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T08:28:18,274 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T08:28:18,277 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T08:28:18,277 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T08:28:18,277 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T08:28:18,277 DEBUG [RS:1;dd3a0865bb23:44073 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228e35e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd3a0865bb23/172.17.0.2:0 2024-11-13T08:28:18,277 DEBUG [RS:0;dd3a0865bb23:35727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fe7da3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd3a0865bb23/172.17.0.2:0 2024-11-13T08:28:18,277 DEBUG [RS:2;dd3a0865bb23:45659 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52397fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd3a0865bb23/172.17.0.2:0 2024-11-13T08:28:18,296 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;dd3a0865bb23:45659 2024-11-13T08:28:18,299 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dd3a0865bb23:44073 2024-11-13T08:28:18,299 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd3a0865bb23:35727 2024-11-13T08:28:18,299 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T08:28:18,299 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T08:28:18,300 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T08:28:18,300 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T08:28:18,300 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T08:28:18,300 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T08:28:18,300 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T08:28:18,300 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T08:28:18,300 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T08:28:18,303 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd3a0865bb23,37177,1731486496596 with port=35727, startcode=1731486497292 2024-11-13T08:28:18,303 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd3a0865bb23,37177,1731486496596 with port=45659, startcode=1731486497432 2024-11-13T08:28:18,303 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd3a0865bb23,37177,1731486496596 with port=44073, startcode=1731486497392 2024-11-13T08:28:18,317 DEBUG [RS:0;dd3a0865bb23:35727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T08:28:18,317 DEBUG [RS:1;dd3a0865bb23:44073 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T08:28:18,317 DEBUG [RS:2;dd3a0865bb23:45659 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T08:28:18,316 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T08:28:18,326 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T08:28:18,334 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T08:28:18,340 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd3a0865bb23,37177,1731486496596 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T08:28:18,349 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd3a0865bb23:0, corePoolSize=5, maxPoolSize=5 2024-11-13T08:28:18,350 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd3a0865bb23:0, corePoolSize=5, maxPoolSize=5 2024-11-13T08:28:18,350 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd3a0865bb23:0, corePoolSize=5, maxPoolSize=5 2024-11-13T08:28:18,350 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd3a0865bb23:0, corePoolSize=5, maxPoolSize=5 2024-11-13T08:28:18,350 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd3a0865bb23:0, corePoolSize=10, maxPoolSize=10 2024-11-13T08:28:18,350 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,351 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd3a0865bb23:0, corePoolSize=2, maxPoolSize=2 2024-11-13T08:28:18,351 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T08:28:18,358 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731486528358 2024-11-13T08:28:18,359 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T08:28:18,360 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T08:28:18,360 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58277, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T08:28:18,360 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55883, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T08:28:18,361 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T08:28:18,362 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T08:28:18,366 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T08:28:18,366 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:18,367 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T08:28:18,367 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T08:28:18,367 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T08:28:18,367 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T08:28:18,368 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T08:28:18,370 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,375 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T08:28:18,375 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T08:28:18,375 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T08:28:18,376 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T08:28:18,376 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T08:28:18,378 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T08:28:18,378 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T08:28:18,380 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd3a0865bb23:0:becomeActiveMaster-HFileCleaner.large.0-1731486498379,5,FailOnTimeoutGroup] 2024-11-13T08:28:18,381 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd3a0865bb23:0:becomeActiveMaster-HFileCleaner.small.0-1731486498381,5,FailOnTimeoutGroup] 2024-11-13T08:28:18,381 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,381 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T08:28:18,397 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,398 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741831_1007 (size=1321) 2024-11-13T08:28:18,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741831_1007 (size=1321) 2024-11-13T08:28:18,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741831_1007 (size=1321) 2024-11-13T08:28:18,402 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T08:28:18,403 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc 2024-11-13T08:28:18,410 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T08:28:18,410 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T08:28:18,410 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T08:28:18,410 WARN [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T08:28:18,410 WARN [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T08:28:18,410 WARN [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T08:28:18,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741832_1008 (size=32) 2024-11-13T08:28:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741832_1008 (size=32) 2024-11-13T08:28:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741832_1008 (size=32) 2024-11-13T08:28:18,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:18,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T08:28:18,424 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T08:28:18,424 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:18,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:18,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T08:28:18,427 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T08:28:18,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:18,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:18,429 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T08:28:18,431 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T08:28:18,431 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:18,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:18,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T08:28:18,435 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T08:28:18,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:18,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:18,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T08:28:18,438 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740 2024-11-13T08:28:18,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740 2024-11-13T08:28:18,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T08:28:18,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T08:28:18,443 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T08:28:18,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T08:28:18,450 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:18,451 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67419358, jitterRate=0.004626721143722534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T08:28:18,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731486498418Initializing all the Stores at 1731486498420 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486498420Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486498421 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486498421Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486498421Cleaning up temporary data from old regions at 1731486498442 (+21 ms)Region opened successfully at 1731486498455 (+13 ms) 2024-11-13T08:28:18,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T08:28:18,455 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T08:28:18,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T08:28:18,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T08:28:18,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T08:28:18,457 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T08:28:18,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731486498455Disabling compacts and flushes for region at 1731486498455Disabling writes for close at 1731486498455Writing region close event to WAL at 1731486498457 (+2 ms)Closed at 1731486498457 2024-11-13T08:28:18,461 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T08:28:18,461 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T08:28:18,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T08:28:18,477 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T08:28:18,480 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T08:28:18,511 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd3a0865bb23,37177,1731486496596 with port=45659, startcode=1731486497432 2024-11-13T08:28:18,511 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd3a0865bb23,37177,1731486496596 with port=44073, startcode=1731486497392 2024-11-13T08:28:18,511 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd3a0865bb23,37177,1731486496596 with port=35727, startcode=1731486497292 2024-11-13T08:28:18,513 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,516 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] master.ServerManager(517): Registering regionserver=dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,525 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:18,525 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] master.ServerManager(517): Registering regionserver=dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:18,525 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc 2024-11-13T08:28:18,526 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38569 2024-11-13T08:28:18,526 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T08:28:18,528 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:18,528 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37177 {}] master.ServerManager(517): Registering regionserver=dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:18,528 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc 2024-11-13T08:28:18,528 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38569 2024-11-13T08:28:18,528 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T08:28:18,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T08:28:18,531 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc 2024-11-13T08:28:18,531 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38569 2024-11-13T08:28:18,531 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T08:28:18,535 DEBUG [RS:2;dd3a0865bb23:45659 {}] zookeeper.ZKUtil(111): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,535 WARN [RS:2;dd3a0865bb23:45659 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T08:28:18,535 INFO [RS:2;dd3a0865bb23:45659 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:18,535 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T08:28:18,536 DEBUG [RS:0;dd3a0865bb23:35727 {}] zookeeper.ZKUtil(111): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:18,536 WARN [RS:0;dd3a0865bb23:35727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T08:28:18,536 INFO [RS:0;dd3a0865bb23:35727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:18,536 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd3a0865bb23,45659,1731486497432] 2024-11-13T08:28:18,537 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd3a0865bb23,35727,1731486497292] 2024-11-13T08:28:18,537 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:18,537 DEBUG [RS:1;dd3a0865bb23:44073 {}] zookeeper.ZKUtil(111): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:18,537 WARN [RS:1;dd3a0865bb23:44073 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T08:28:18,537 INFO [RS:1;dd3a0865bb23:44073 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:18,537 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:18,538 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd3a0865bb23,44073,1731486497392] 2024-11-13T08:28:18,563 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T08:28:18,563 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T08:28:18,563 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T08:28:18,579 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T08:28:18,579 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T08:28:18,579 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T08:28:18,584 INFO [RS:2;dd3a0865bb23:45659 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T08:28:18,584 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,584 INFO [RS:0;dd3a0865bb23:35727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T08:28:18,584 INFO [RS:1;dd3a0865bb23:44073 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T08:28:18,584 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,584 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,585 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T08:28:18,585 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T08:28:18,585 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T08:28:18,591 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T08:28:18,591 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T08:28:18,591 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T08:28:18,592 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,592 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,592 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,592 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,592 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,592 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,592 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,592 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,592 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,592 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0, corePoolSize=2, maxPoolSize=2 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0, corePoolSize=2, maxPoolSize=2 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0, corePoolSize=2, maxPoolSize=2 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,593 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd3a0865bb23:0, corePoolSize=3, maxPoolSize=3 2024-11-13T08:28:18,594 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:2;dd3a0865bb23:45659 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd3a0865bb23:0, corePoolSize=3, maxPoolSize=3 2024-11-13T08:28:18,594 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd3a0865bb23:0, corePoolSize=3, maxPoolSize=3 2024-11-13T08:28:18,594 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:0;dd3a0865bb23:35727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd3a0865bb23:0, corePoolSize=3, maxPoolSize=3 2024-11-13T08:28:18,594 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd3a0865bb23:0, corePoolSize=1, maxPoolSize=1 2024-11-13T08:28:18,594 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd3a0865bb23:0, corePoolSize=3, maxPoolSize=3 2024-11-13T08:28:18,594 DEBUG [RS:1;dd3a0865bb23:44073 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd3a0865bb23:0, corePoolSize=3, maxPoolSize=3 2024-11-13T08:28:18,600 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,600 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,600 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,45659,1731486497432-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,35727,1731486497292-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,601 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,44073,1731486497392-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T08:28:18,621 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T08:28:18,621 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T08:28:18,623 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,44073,1731486497392-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,623 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,45659,1731486497432-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,623 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,623 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,623 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.Replication(171): dd3a0865bb23,44073,1731486497392 started 2024-11-13T08:28:18,623 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.Replication(171): dd3a0865bb23,45659,1731486497432 started 2024-11-13T08:28:18,625 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T08:28:18,625 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,35727,1731486497292-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,625 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,625 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.Replication(171): dd3a0865bb23,35727,1731486497292 started 2024-11-13T08:28:18,631 WARN [dd3a0865bb23:37177 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T08:28:18,641 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,641 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,641 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1482): Serving as dd3a0865bb23,44073,1731486497392, RpcServer on dd3a0865bb23/172.17.0.2:44073, sessionid=0x10110711ec40002 2024-11-13T08:28:18,641 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1482): Serving as dd3a0865bb23,45659,1731486497432, RpcServer on dd3a0865bb23/172.17.0.2:45659, sessionid=0x10110711ec40003 2024-11-13T08:28:18,642 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T08:28:18,642 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T08:28:18,642 DEBUG [RS:1;dd3a0865bb23:44073 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:18,642 DEBUG [RS:2;dd3a0865bb23:45659 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,642 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd3a0865bb23,44073,1731486497392' 2024-11-13T08:28:18,642 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd3a0865bb23,45659,1731486497432' 2024-11-13T08:28:18,642 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T08:28:18,642 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T08:28:18,643 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T08:28:18,644 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T08:28:18,644 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T08:28:18,644 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T08:28:18,644 DEBUG [RS:2;dd3a0865bb23:45659 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,644 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd3a0865bb23,45659,1731486497432' 2024-11-13T08:28:18,644 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T08:28:18,644 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T08:28:18,645 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T08:28:18,645 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T08:28:18,645 DEBUG [RS:1;dd3a0865bb23:44073 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:18,645 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd3a0865bb23,44073,1731486497392' 2024-11-13T08:28:18,645 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T08:28:18,645 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T08:28:18,645 DEBUG [RS:2;dd3a0865bb23:45659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T08:28:18,645 INFO [RS:2;dd3a0865bb23:45659 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T08:28:18,645 INFO [RS:2;dd3a0865bb23:45659 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T08:28:18,646 DEBUG [RS:1;dd3a0865bb23:44073 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T08:28:18,646 INFO [RS:1;dd3a0865bb23:44073 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T08:28:18,646 INFO [RS:1;dd3a0865bb23:44073 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T08:28:18,648 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:18,648 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1482): Serving as dd3a0865bb23,35727,1731486497292, RpcServer on dd3a0865bb23/172.17.0.2:35727, sessionid=0x10110711ec40001 2024-11-13T08:28:18,648 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T08:28:18,648 DEBUG [RS:0;dd3a0865bb23:35727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:18,648 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd3a0865bb23,35727,1731486497292' 2024-11-13T08:28:18,648 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T08:28:18,649 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T08:28:18,650 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T08:28:18,650 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T08:28:18,650 DEBUG [RS:0;dd3a0865bb23:35727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:18,650 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd3a0865bb23,35727,1731486497292' 2024-11-13T08:28:18,650 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T08:28:18,650 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T08:28:18,651 DEBUG [RS:0;dd3a0865bb23:35727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T08:28:18,651 INFO [RS:0;dd3a0865bb23:35727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T08:28:18,651 INFO [RS:0;dd3a0865bb23:35727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T08:28:18,751 INFO [RS:1;dd3a0865bb23:44073 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:18,751 INFO [RS:2;dd3a0865bb23:45659 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:18,752 INFO [RS:0;dd3a0865bb23:35727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:18,754 INFO [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd3a0865bb23%2C45659%2C1731486497432, suffix=, logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432, archiveDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs, maxLogs=32 2024-11-13T08:28:18,754 INFO [RS:1;dd3a0865bb23:44073 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd3a0865bb23%2C44073%2C1731486497392, suffix=, logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392, archiveDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs, maxLogs=32 2024-11-13T08:28:18,755 INFO [RS:0;dd3a0865bb23:35727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd3a0865bb23%2C35727%2C1731486497292, suffix=, logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,35727,1731486497292, archiveDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs, maxLogs=32 2024-11-13T08:28:18,773 DEBUG [RS:2;dd3a0865bb23:45659 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432/dd3a0865bb23%2C45659%2C1731486497432.1731486498760, exclude list is [], retry=0 2024-11-13T08:28:18,775 DEBUG [RS:1;dd3a0865bb23:44073 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392/dd3a0865bb23%2C44073%2C1731486497392.1731486498760, exclude list is [], retry=0 2024-11-13T08:28:18,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:18,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:18,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:18,780 DEBUG [RS:0;dd3a0865bb23:35727 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,35727,1731486497292/dd3a0865bb23%2C35727%2C1731486497292.1731486498760, exclude list is [], retry=0 2024-11-13T08:28:18,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:18,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:18,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:18,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:18,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:18,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:18,797 INFO [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432/dd3a0865bb23%2C45659%2C1731486497432.1731486498760 2024-11-13T08:28:18,798 INFO [RS:1;dd3a0865bb23:44073 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 2024-11-13T08:28:18,798 DEBUG [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:18,799 INFO [RS:0;dd3a0865bb23:35727 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,35727,1731486497292/dd3a0865bb23%2C35727%2C1731486497292.1731486498760 2024-11-13T08:28:18,800 DEBUG [RS:1;dd3a0865bb23:44073 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:18,802 DEBUG [RS:0;dd3a0865bb23:35727 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:18,883 DEBUG [dd3a0865bb23:37177 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T08:28:18,891 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(204): Hosts are {dd3a0865bb23=0} racks are {/default-rack=0} 2024-11-13T08:28:18,897 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T08:28:18,897 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T08:28:18,897 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T08:28:18,897 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T08:28:18,897 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T08:28:18,897 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T08:28:18,897 INFO [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T08:28:18,897 INFO [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T08:28:18,897 INFO [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T08:28:18,898 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T08:28:18,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:18,911 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd3a0865bb23,45659,1731486497432, state=OPENING 2024-11-13T08:28:18,916 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T08:28:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:18,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:18,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:18,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:18,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:18,921 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T08:28:18,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd3a0865bb23,45659,1731486497432}] 2024-11-13T08:28:19,100 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T08:28:19,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42215, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T08:28:19,114 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T08:28:19,115 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:19,115 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T08:28:19,118 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd3a0865bb23%2C45659%2C1731486497432.meta, suffix=.meta, logDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432, archiveDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs, maxLogs=32 2024-11-13T08:28:19,134 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432/dd3a0865bb23%2C45659%2C1731486497432.meta.1731486499120.meta, exclude list is [], retry=0 2024-11-13T08:28:19,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:19,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:19,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:19,141 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,45659,1731486497432/dd3a0865bb23%2C45659%2C1731486497432.meta.1731486499120.meta 2024-11-13T08:28:19,142 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:19,142 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:19,143 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T08:28:19,146 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T08:28:19,151 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T08:28:19,155 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T08:28:19,156 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:19,156 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T08:28:19,156 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T08:28:19,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T08:28:19,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T08:28:19,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:19,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:19,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T08:28:19,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T08:28:19,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:19,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:19,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T08:28:19,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T08:28:19,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:19,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:19,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T08:28:19,167 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T08:28:19,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:19,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T08:28:19,168 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T08:28:19,169 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740 2024-11-13T08:28:19,172 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740 2024-11-13T08:28:19,174 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T08:28:19,174 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T08:28:19,175 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T08:28:19,177 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T08:28:19,179 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61669044, jitterRate=-0.08105963468551636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T08:28:19,179 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T08:28:19,180 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731486499156Writing region info on filesystem at 1731486499157 (+1 ms)Initializing all the Stores at 1731486499158 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486499159 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486499159Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486499159Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731486499159Cleaning up temporary data from old regions at 1731486499174 (+15 ms)Running coprocessor post-open hooks at 1731486499179 (+5 ms)Region opened successfully at 1731486499180 (+1 ms) 2024-11-13T08:28:19,187 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731486499091 2024-11-13T08:28:19,198 DEBUG [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T08:28:19,199 INFO [RS_OPEN_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T08:28:19,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:19,202 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd3a0865bb23,45659,1731486497432, state=OPEN 2024-11-13T08:28:19,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T08:28:19,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T08:28:19,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T08:28:19,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T08:28:19,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:19,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:19,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:19,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T08:28:19,205 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:19,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T08:28:19,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd3a0865bb23,45659,1731486497432 in 283 msec 2024-11-13T08:28:19,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T08:28:19,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 744 msec 2024-11-13T08:28:19,218 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T08:28:19,219 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T08:28:19,242 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T08:28:19,243 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd3a0865bb23,45659,1731486497432, seqNum=-1] 2024-11-13T08:28:19,263 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T08:28:19,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35373, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T08:28:19,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0140 sec 2024-11-13T08:28:19,285 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731486499285, completionTime=-1 2024-11-13T08:28:19,289 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T08:28:19,289 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T08:28:19,320 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T08:28:19,320 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731486559320 2024-11-13T08:28:19,320 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731486619320 2024-11-13T08:28:19,320 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-13T08:28:19,322 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T08:28:19,329 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,37177,1731486496596-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,329 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,37177,1731486496596-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,329 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,37177,1731486496596-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,331 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd3a0865bb23:37177, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,331 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,332 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,338 DEBUG [master/dd3a0865bb23:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T08:28:19,359 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.843sec 2024-11-13T08:28:19,360 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T08:28:19,361 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T08:28:19,362 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T08:28:19,363 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T08:28:19,363 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T08:28:19,364 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,37177,1731486496596-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T08:28:19,365 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,37177,1731486496596-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T08:28:19,370 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T08:28:19,370 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T08:28:19,371 INFO [master/dd3a0865bb23:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd3a0865bb23,37177,1731486496596-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:19,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7df72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T08:28:19,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd3a0865bb23,37177,-1 for getting cluster id 2024-11-13T08:28:19,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T08:28:19,387 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '976bee63-cb40-472b-8f8e-0f050f8e2d48' 2024-11-13T08:28:19,389 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T08:28:19,389 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "976bee63-cb40-472b-8f8e-0f050f8e2d48" 2024-11-13T08:28:19,391 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582136bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T08:28:19,391 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd3a0865bb23,37177,-1] 2024-11-13T08:28:19,394 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T08:28:19,396 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:19,397 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T08:28:19,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11b5a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T08:28:19,400 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T08:28:19,406 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd3a0865bb23,45659,1731486497432, seqNum=-1] 2024-11-13T08:28:19,407 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T08:28:19,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T08:28:19,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:19,431 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38569/hbase 2024-11-13T08:28:19,447 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=361, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=2098 2024-11-13T08:28:19,466 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:19,470 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:19,470 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:19,474 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73202368, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-73202368, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:19,491 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73202368/hregion-73202368.1731486499476, exclude list is [], retry=0 2024-11-13T08:28:19,494 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:19,495 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:19,495 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:19,498 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73202368/hregion-73202368.1731486499476 2024-11-13T08:28:19,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:19,499 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c3913b974eff8f70adf932560636d3e2, NAME => 'testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:19,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741838_1014 (size=64) 2024-11-13T08:28:19,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741838_1014 (size=64) 2024-11-13T08:28:19,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741838_1014 (size=64) 2024-11-13T08:28:19,513 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:19,515 INFO [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,517 INFO [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3913b974eff8f70adf932560636d3e2 columnFamilyName a 2024-11-13T08:28:19,517 DEBUG [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:19,518 INFO [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] regionserver.HStore(327): Store=c3913b974eff8f70adf932560636d3e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:19,518 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,523 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,526 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:19,527 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c3913b974eff8f70adf932560636d3e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72316715, jitterRate=0.07760302722454071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:19,529 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c3913b974eff8f70adf932560636d3e2: Writing region info on filesystem at 1731486499513Initializing all the Stores at 1731486499514 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486499514Cleaning up temporary data from old regions at 1731486499521 (+7 ms)Region opened successfully at 1731486499529 (+8 ms) 2024-11-13T08:28:19,529 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c3913b974eff8f70adf932560636d3e2, disabling compactions & flushes 2024-11-13T08:28:19,529 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2. 2024-11-13T08:28:19,529 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2. 2024-11-13T08:28:19,529 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2. after waiting 0 ms 2024-11-13T08:28:19,529 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2. 2024-11-13T08:28:19,529 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2. 2024-11-13T08:28:19,530 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c3913b974eff8f70adf932560636d3e2: Waiting for close lock at 1731486499529Disabling compacts and flushes for region at 1731486499529Disabling writes for close at 1731486499529Writing region close event to WAL at 1731486499529Closed at 1731486499529 2024-11-13T08:28:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741837_1013 (size=93) 2024-11-13T08:28:19,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741837_1013 (size=93) 2024-11-13T08:28:19,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741837_1013 (size=93) 2024-11-13T08:28:19,543 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:19,543 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73202368:(num 1731486499476) 2024-11-13T08:28:19,545 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T08:28:19,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741839_1015 (size=369) 2024-11-13T08:28:19,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741839_1015 (size=369) 2024-11-13T08:28:19,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741839_1015 (size=369) 2024-11-13T08:28:19,564 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T08:28:19,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741840_1016 (size=231) 2024-11-13T08:28:19,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741840_1016 (size=231) 2024-11-13T08:28:19,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741840_1016 (size=231) 2024-11-13T08:28:19,605 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1, size=369 (369bytes) 2024-11-13T08:28:19,606 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T08:28:19,606 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T08:28:19,607 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1 2024-11-13T08:28:19,612 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1 after 4ms 2024-11-13T08:28:19,620 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1 took 16ms 2024-11-13T08:28:19,623 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1 so closing down 2024-11-13T08:28:19,626 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T08:28:19,628 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T08:28:19,629 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:19,629 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:19,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741841_1017 (size=369) 2024-11-13T08:28:19,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741841_1017 (size=369) 2024-11-13T08:28:19,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741841_1017 (size=369) 2024-11-13T08:28:19,643 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:19,646 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002 2024-11-13T08:28:19,650 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T08:28:19,650 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1, journal: Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1, size=369 (369bytes) at 1731486499605Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1 so closing down at 1731486499623 (+18 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000001-wal-1.temp at 1731486499628 (+5 ms)3 split writer threads finished at 1731486499630 (+2 ms)Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731486499643 (+13 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002 at 1731486499646 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731486499650 (+4 ms) 2024-11-13T08:28:19,666 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2, size=231 (231bytes) 2024-11-13T08:28:19,666 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2 2024-11-13T08:28:19,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2 after 1ms 2024-11-13T08:28:19,671 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2 took 5ms 2024-11-13T08:28:19,673 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2 so closing down 2024-11-13T08:28:19,673 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:19,676 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T08:28:19,677 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T08:28:19,677 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:19,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741842_1018 (size=231) 2024-11-13T08:28:19,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741842_1018 (size=231) 2024-11-13T08:28:19,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741842_1018 (size=231) 2024-11-13T08:28:19,686 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:19,692 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-13T08:28:19,694 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T08:28:19,694 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2, journal: Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2, size=231 (231bytes) at 1731486499666Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2 so closing down at 1731486499673 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002-wal-2.temp at 1731486499677 (+4 ms)3 split writer threads finished at 1731486499677Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731486499686 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731486499694 (+8 ms) 2024-11-13T08:28:19,694 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:19,696 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:19,713 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal.1731486499697, exclude list is [], retry=0 2024-11-13T08:28:19,717 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:19,718 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:19,718 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:19,721 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal.1731486499697 2024-11-13T08:28:19,721 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:19,722 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c3913b974eff8f70adf932560636d3e2, NAME => 'testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:19,722 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:19,722 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,722 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,724 INFO [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,726 INFO [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3913b974eff8f70adf932560636d3e2 columnFamilyName a 2024-11-13T08:28:19,726 DEBUG [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:19,727 INFO [StoreOpener-c3913b974eff8f70adf932560636d3e2-1 {}] regionserver.HStore(327): Store=c3913b974eff8f70adf932560636d3e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:19,727 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,728 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,733 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002 2024-11-13T08:28:19,744 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002 2024-11-13T08:28:19,748 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c3913b974eff8f70adf932560636d3e2 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T08:28:19,803 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/.tmp/a/a2a4f1a9fc5448a381e44a33c4c56f38 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731486499543/Put/seqid=0 2024-11-13T08:28:19,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741844_1020 (size=5170) 2024-11-13T08:28:19,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741844_1020 (size=5170) 2024-11-13T08:28:19,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741844_1020 (size=5170) 2024-11-13T08:28:19,818 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/.tmp/a/a2a4f1a9fc5448a381e44a33c4c56f38 2024-11-13T08:28:19,864 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/.tmp/a/a2a4f1a9fc5448a381e44a33c4c56f38 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/a/a2a4f1a9fc5448a381e44a33c4c56f38 2024-11-13T08:28:19,875 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/a/a2a4f1a9fc5448a381e44a33c4c56f38, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T08:28:19,880 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c3913b974eff8f70adf932560636d3e2 in 131ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T08:28:19,881 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/0000000000000000002 2024-11-13T08:28:19,882 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,882 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,885 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c3913b974eff8f70adf932560636d3e2 2024-11-13T08:28:19,888 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/c3913b974eff8f70adf932560636d3e2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T08:28:19,890 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c3913b974eff8f70adf932560636d3e2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60022142, jitterRate=-0.10560038685798645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:19,891 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c3913b974eff8f70adf932560636d3e2: Writing region info on filesystem at 1731486499722Initializing all the Stores at 1731486499724 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486499724Obtaining lock to block concurrent updates at 1731486499748 (+24 ms)Preparing flush snapshotting stores in c3913b974eff8f70adf932560636d3e2 at 1731486499748Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731486499752 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731486499467.c3913b974eff8f70adf932560636d3e2. at 1731486499752Flushing c3913b974eff8f70adf932560636d3e2/a: creating writer at 1731486499753 (+1 ms)Flushing c3913b974eff8f70adf932560636d3e2/a: appending metadata at 1731486499794 (+41 ms)Flushing c3913b974eff8f70adf932560636d3e2/a: closing flushed file at 1731486499796 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2069a8fd: reopening flushed file at 1731486499862 (+66 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c3913b974eff8f70adf932560636d3e2 in 131ms, sequenceid=2, compaction requested=false; wal=null at 1731486499880 (+18 ms)Cleaning up temporary data from old regions at 1731486499882 (+2 ms)Region opened successfully at 1731486499891 (+9 ms) 2024-11-13T08:28:19,948 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 361) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53528 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:38320 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:38282 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38569/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53692 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:37138 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:37006 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=2010 (was 2098) 2024-11-13T08:28:19,963 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=2009 2024-11-13T08:28:19,990 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:19,992 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:19,993 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:19,996 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77723000, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-77723000, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:20,010 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77723000/hregion-77723000.1731486499997, exclude list is [], retry=0 2024-11-13T08:28:20,014 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:20,014 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:20,014 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:20,017 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77723000/hregion-77723000.1731486499997 2024-11-13T08:28:20,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:20,018 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0064a97631313af5a89a6922a57335ad, NAME => 'testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:20,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741846_1022 (size=64) 2024-11-13T08:28:20,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741846_1022 (size=64) 2024-11-13T08:28:20,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741846_1022 (size=64) 2024-11-13T08:28:20,035 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:20,037 INFO [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,039 INFO [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0064a97631313af5a89a6922a57335ad columnFamilyName a 2024-11-13T08:28:20,039 DEBUG [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:20,040 INFO [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] regionserver.HStore(327): Store=0064a97631313af5a89a6922a57335ad/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:20,040 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,041 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,042 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,042 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,042 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,044 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,047 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:20,048 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0064a97631313af5a89a6922a57335ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75249572, jitterRate=0.12130600214004517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:20,048 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0064a97631313af5a89a6922a57335ad: Writing region info on filesystem at 1731486500035Initializing all the Stores at 1731486500036 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486500036Cleaning up temporary data from old regions at 1731486500042 (+6 ms)Region opened successfully at 1731486500048 (+6 ms) 2024-11-13T08:28:20,048 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0064a97631313af5a89a6922a57335ad, disabling compactions & flushes 2024-11-13T08:28:20,048 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad. 2024-11-13T08:28:20,048 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad. 2024-11-13T08:28:20,048 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad. after waiting 0 ms 2024-11-13T08:28:20,048 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad. 2024-11-13T08:28:20,048 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad. 2024-11-13T08:28:20,049 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0064a97631313af5a89a6922a57335ad: Waiting for close lock at 1731486500048Disabling compacts and flushes for region at 1731486500048Disabling writes for close at 1731486500048Writing region close event to WAL at 1731486500048Closed at 1731486500048 2024-11-13T08:28:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741845_1021 (size=93) 2024-11-13T08:28:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741845_1021 (size=93) 2024-11-13T08:28:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741845_1021 (size=93) 2024-11-13T08:28:20,055 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:20,056 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77723000:(num 1731486499997) 2024-11-13T08:28:20,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T08:28:20,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741847_1023 (size=369) 2024-11-13T08:28:20,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741847_1023 (size=369) 2024-11-13T08:28:20,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741847_1023 (size=369) 2024-11-13T08:28:20,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T08:28:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741848_1024 (size=231) 2024-11-13T08:28:20,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741848_1024 (size=231) 2024-11-13T08:28:20,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741848_1024 (size=231) 2024-11-13T08:28:20,107 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2, size=231 (231bytes) 2024-11-13T08:28:20,107 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2 2024-11-13T08:28:20,108 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2 after 1ms 2024-11-13T08:28:20,112 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2 took 5ms 2024-11-13T08:28:20,114 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2 so closing down 2024-11-13T08:28:20,115 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:20,117 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T08:28:20,118 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T08:28:20,119 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741849_1025 (size=231) 2024-11-13T08:28:20,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741849_1025 (size=231) 2024-11-13T08:28:20,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741849_1025 (size=231) 2024-11-13T08:28:20,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:20,131 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 2024-11-13T08:28:20,131 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T08:28:20,131 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2, journal: Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2, size=231 (231bytes) at 1731486500107Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2 so closing down at 1731486500115 (+8 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002-wal-2.temp at 1731486500118 (+3 ms)3 split writer threads finished at 1731486500119 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731486500128 (+9 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 at 1731486500131 (+3 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731486500131 2024-11-13T08:28:20,146 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1, size=369 (369bytes) 2024-11-13T08:28:20,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1 2024-11-13T08:28:20,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1 after 1ms 2024-11-13T08:28:20,151 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1 took 5ms 2024-11-13T08:28:20,154 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1 so closing down 2024-11-13T08:28:20,154 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:20,156 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T08:28:20,158 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T08:28:20,158 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741850_1026 (size=369) 2024-11-13T08:28:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741850_1026 (size=369) 2024-11-13T08:28:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741850_1026 (size=369) 2024-11-13T08:28:20,168 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:20,175 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002, length=231 2024-11-13T08:28:20,178 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 2024-11-13T08:28:20,178 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T08:28:20,178 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1, journal: Splitting hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1, size=369 (369bytes) at 1731486500146Finishing writing output for hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1 so closing down at 1731486500154 (+8 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000001-wal-1.temp at 1731486500158 (+4 ms)3 split writer threads finished at 1731486500158Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731486500168 (+10 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 at 1731486500178 (+10 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731486500178 2024-11-13T08:28:20,178 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:20,180 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:20,195 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal.1731486500181, exclude list is [], retry=0 2024-11-13T08:28:20,199 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:20,199 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:20,200 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:20,202 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal.1731486500181 2024-11-13T08:28:20,203 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:20,203 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0064a97631313af5a89a6922a57335ad, NAME => 'testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:20,203 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:20,203 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,203 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,205 INFO [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,207 INFO [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0064a97631313af5a89a6922a57335ad columnFamilyName a 2024-11-13T08:28:20,207 DEBUG [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:20,208 INFO [StoreOpener-0064a97631313af5a89a6922a57335ad-1 {}] regionserver.HStore(327): Store=0064a97631313af5a89a6922a57335ad/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:20,208 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,212 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 2024-11-13T08:28:20,217 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 2024-11-13T08:28:20,217 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0064a97631313af5a89a6922a57335ad 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T08:28:20,234 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/.tmp/a/2f2c693b14e94ad284fe031b45c8d894 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731486500056/Put/seqid=0 2024-11-13T08:28:20,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741852_1028 (size=5170) 2024-11-13T08:28:20,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741852_1028 (size=5170) 2024-11-13T08:28:20,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741852_1028 (size=5170) 2024-11-13T08:28:20,243 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/.tmp/a/2f2c693b14e94ad284fe031b45c8d894 2024-11-13T08:28:20,251 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/.tmp/a/2f2c693b14e94ad284fe031b45c8d894 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/a/2f2c693b14e94ad284fe031b45c8d894 2024-11-13T08:28:20,260 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/a/2f2c693b14e94ad284fe031b45c8d894, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T08:28:20,260 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0064a97631313af5a89a6922a57335ad in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T08:28:20,261 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/0000000000000000002 2024-11-13T08:28:20,261 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,262 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,264 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0064a97631313af5a89a6922a57335ad 2024-11-13T08:28:20,266 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/0064a97631313af5a89a6922a57335ad/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T08:28:20,268 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0064a97631313af5a89a6922a57335ad; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59247362, jitterRate=-0.11714550852775574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:20,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0064a97631313af5a89a6922a57335ad: Writing region info on filesystem at 1731486500203Initializing all the Stores at 1731486500205 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486500205Obtaining lock to block concurrent updates at 1731486500217 (+12 ms)Preparing flush snapshotting stores in 0064a97631313af5a89a6922a57335ad at 1731486500217Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731486500217Flushing stores of testReplayEditsWrittenIntoWAL,,1731486499991.0064a97631313af5a89a6922a57335ad. at 1731486500217Flushing 0064a97631313af5a89a6922a57335ad/a: creating writer at 1731486500218 (+1 ms)Flushing 0064a97631313af5a89a6922a57335ad/a: appending metadata at 1731486500233 (+15 ms)Flushing 0064a97631313af5a89a6922a57335ad/a: closing flushed file at 1731486500233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@608a07a3: reopening flushed file at 1731486500250 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0064a97631313af5a89a6922a57335ad in 43ms, sequenceid=2, compaction requested=false; wal=null at 1731486500260 (+10 ms)Cleaning up temporary data from old regions at 1731486500262 (+2 ms)Region opened successfully at 1731486500268 (+6 ms) 2024-11-13T08:28:20,284 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53770 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53528 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:38282 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:38386 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:37194 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:37006 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=2004 (was 2009) 2024-11-13T08:28:20,295 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=2003 2024-11-13T08:28:20,310 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:20,312 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:20,313 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:20,316 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-09901665, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-09901665, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:20,329 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-09901665/hregion-09901665.1731486500316, exclude list is [], retry=0 2024-11-13T08:28:20,333 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:20,333 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:20,333 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:20,336 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-09901665/hregion-09901665.1731486500316 2024-11-13T08:28:20,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:20,336 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d0fe7a87c90da5adf1c927ac8b476831, NAME => 'testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741854_1030 (size=64) 2024-11-13T08:28:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741854_1030 (size=64) 2024-11-13T08:28:20,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741854_1030 (size=64) 2024-11-13T08:28:20,347 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:20,349 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,351 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0fe7a87c90da5adf1c927ac8b476831 columnFamilyName a 2024-11-13T08:28:20,351 DEBUG [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:20,351 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(327): Store=d0fe7a87c90da5adf1c927ac8b476831/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:20,352 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,353 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0fe7a87c90da5adf1c927ac8b476831 columnFamilyName b 2024-11-13T08:28:20,353 DEBUG [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:20,354 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(327): Store=d0fe7a87c90da5adf1c927ac8b476831/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:20,354 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,355 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0fe7a87c90da5adf1c927ac8b476831 columnFamilyName c 2024-11-13T08:28:20,355 DEBUG [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:20,356 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(327): Store=d0fe7a87c90da5adf1c927ac8b476831/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:20,356 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,357 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,358 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,359 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,359 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,360 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:20,362 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:20,365 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:20,365 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0fe7a87c90da5adf1c927ac8b476831; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67230310, jitterRate=0.0018096864223480225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:20,366 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0fe7a87c90da5adf1c927ac8b476831: Writing region info on filesystem at 1731486500348Initializing all the Stores at 1731486500349 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486500349Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486500349Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486500349Cleaning up temporary data from old regions at 1731486500359 (+10 ms)Region opened successfully at 1731486500366 (+7 ms) 2024-11-13T08:28:20,368 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0fe7a87c90da5adf1c927ac8b476831, disabling compactions & flushes 2024-11-13T08:28:20,368 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:20,368 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:20,368 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. after waiting 0 ms 2024-11-13T08:28:20,368 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:20,369 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:20,369 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0fe7a87c90da5adf1c927ac8b476831: Waiting for close lock at 1731486500368Disabling compacts and flushes for region at 1731486500368Disabling writes for close at 1731486500368Writing region close event to WAL at 1731486500368Closed at 1731486500368 2024-11-13T08:28:20,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741853_1029 (size=93) 2024-11-13T08:28:20,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741853_1029 (size=93) 2024-11-13T08:28:20,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741853_1029 (size=93) 2024-11-13T08:28:20,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:20,376 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-09901665:(num 1731486500316) 2024-11-13T08:28:20,376 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:20,378 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:20,392 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379, exclude list is [], retry=0 2024-11-13T08:28:20,395 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:20,395 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:20,396 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:20,398 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 2024-11-13T08:28:20,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:20,581 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379, size=0 (0bytes) 2024-11-13T08:28:20,581 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 might be still open, length is 0 2024-11-13T08:28:20,581 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 2024-11-13T08:28:20,582 WARN [IPC Server handler 0 on default port 38569 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T08:28:20,582 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 after 1ms 2024-11-13T08:28:21,529 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53798 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34057:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53798 dst: /127.0.0.1:34057 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34057 remote=/127.0.0.1:53798]. Total timeout mills is 60000, 59017 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:21,530 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:38428 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38428 dst: /127.0.0.1:34115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:21,530 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:37234 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44221:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37234 dst: /127.0.0.1:44221 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:21,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741855_1032 (size=470675) 2024-11-13T08:28:21,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741855_1032 (size=470675) 2024-11-13T08:28:21,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741855_1032 (size=470675) 2024-11-13T08:28:24,583 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 after 4002ms 2024-11-13T08:28:24,588 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 took 4008ms 2024-11-13T08:28:24,593 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731486500379.temp 2024-11-13T08:28:24,598 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000000001-wal.1731486500379.temp 2024-11-13T08:28:24,659 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T08:28:24,665 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379; continuing. 2024-11-13T08:28:24,665 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 so closing down 2024-11-13T08:28:24,665 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:24,665 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:24,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741856_1033 (size=470683) 2024-11-13T08:28:24,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741856_1033 (size=470683) 2024-11-13T08:28:24,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741856_1033 (size=470683) 2024-11-13T08:28:24,677 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000000001-wal.1731486500379.temp (wrote 3002 edits, skipped 0 edits in 30 ms) 2024-11-13T08:28:24,679 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000000001-wal.1731486500379.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000003002 2024-11-13T08:28:24,680 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T08:28:24,680 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379, journal: Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379, size=0 (0bytes) at 1731486500581Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000000001-wal.1731486500379.temp at 1731486504598 (+4017 ms)Split 1024 edits, skipped 0 edits. at 1731486504634 (+36 ms)Split 2048 edits, skipped 0 edits. at 1731486504652 (+18 ms)Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 so closing down at 1731486504665 (+13 ms)3 split writer threads finished at 1731486504665Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000000001-wal.1731486500379.temp (wrote 3002 edits, skipped 0 edits in 30 ms) at 1731486504677 (+12 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000000001-wal.1731486500379.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000003002 at 1731486504680 (+3 ms)Processed 3002 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379, size=0, length=0, corrupted=false, cancelled=false at 1731486504680 2024-11-13T08:28:24,682 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486500379 2024-11-13T08:28:24,684 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000003002 2024-11-13T08:28:24,684 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:24,687 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:24,706 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486504688, exclude list is [], retry=0 2024-11-13T08:28:24,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:24,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:24,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:24,722 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486504688 2024-11-13T08:28:24,723 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:24,723 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:24,724 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T08:28:24,725 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:24,726 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0fe7a87c90da5adf1c927ac8b476831 columnFamilyName a 2024-11-13T08:28:24,726 DEBUG [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:24,727 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(327): Store=d0fe7a87c90da5adf1c927ac8b476831/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:24,727 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:24,728 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0fe7a87c90da5adf1c927ac8b476831 columnFamilyName b 2024-11-13T08:28:24,728 DEBUG [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:24,729 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(327): Store=d0fe7a87c90da5adf1c927ac8b476831/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:24,729 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:24,730 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0fe7a87c90da5adf1c927ac8b476831 columnFamilyName c 2024-11-13T08:28:24,730 DEBUG [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:24,730 INFO [StoreOpener-d0fe7a87c90da5adf1c927ac8b476831-1 {}] regionserver.HStore(327): Store=d0fe7a87c90da5adf1c927ac8b476831/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:24,731 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:24,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:24,733 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:24,734 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000003002 2024-11-13T08:28:24,765 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T08:28:25,127 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0fe7a87c90da5adf1c927ac8b476831 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T08:28:25,170 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/a/94b76ed812c940b890120da63b440424 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731486500403/Put/seqid=0 2024-11-13T08:28:25,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741858_1035 (size=50463) 2024-11-13T08:28:25,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741858_1035 (size=50463) 2024-11-13T08:28:25,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741858_1035 (size=50463) 2024-11-13T08:28:25,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/a/94b76ed812c940b890120da63b440424 2024-11-13T08:28:25,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/a/94b76ed812c940b890120da63b440424 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/a/94b76ed812c940b890120da63b440424 2024-11-13T08:28:25,199 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/a/94b76ed812c940b890120da63b440424, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T08:28:25,199 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for d0fe7a87c90da5adf1c927ac8b476831 in 72ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T08:28:25,214 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T08:28:25,215 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0fe7a87c90da5adf1c927ac8b476831 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T08:28:25,225 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/a/c696aa02ac954e48818cc0b37a4459dc is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731486500438/Put/seqid=0 2024-11-13T08:28:25,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741859_1036 (size=20072) 2024-11-13T08:28:25,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741859_1036 (size=20072) 2024-11-13T08:28:25,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741859_1036 (size=20072) 2024-11-13T08:28:25,238 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/a/c696aa02ac954e48818cc0b37a4459dc 2024-11-13T08:28:25,269 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/b/7c4594f1b4fe4a89b1b37d737a391130 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731486500458/Put/seqid=0 2024-11-13T08:28:25,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741860_1037 (size=35835) 2024-11-13T08:28:25,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741860_1037 (size=35835) 2024-11-13T08:28:25,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741860_1037 (size=35835) 2024-11-13T08:28:25,281 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/b/7c4594f1b4fe4a89b1b37d737a391130 2024-11-13T08:28:25,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/a/c696aa02ac954e48818cc0b37a4459dc as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/a/c696aa02ac954e48818cc0b37a4459dc 2024-11-13T08:28:25,296 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/a/c696aa02ac954e48818cc0b37a4459dc, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T08:28:25,297 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/b/7c4594f1b4fe4a89b1b37d737a391130 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/b/7c4594f1b4fe4a89b1b37d737a391130 2024-11-13T08:28:25,304 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/b/7c4594f1b4fe4a89b1b37d737a391130, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T08:28:25,304 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for d0fe7a87c90da5adf1c927ac8b476831 in 90ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T08:28:25,313 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T08:28:25,313 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0fe7a87c90da5adf1c927ac8b476831 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T08:28:25,321 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/b/9118cadcfcf8403ebbb463f433e8468b is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731486500479/Put/seqid=0 2024-11-13T08:28:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741861_1038 (size=35082) 2024-11-13T08:28:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741861_1038 (size=35082) 2024-11-13T08:28:25,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741861_1038 (size=35082) 2024-11-13T08:28:25,332 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/b/9118cadcfcf8403ebbb463f433e8468b 2024-11-13T08:28:25,356 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/c/da5a4bc078414f2e8bec6a074ecb2944 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731486500505/Put/seqid=0 2024-11-13T08:28:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741862_1039 (size=20825) 2024-11-13T08:28:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741862_1039 (size=20825) 2024-11-13T08:28:25,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741862_1039 (size=20825) 2024-11-13T08:28:25,365 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/c/da5a4bc078414f2e8bec6a074ecb2944 2024-11-13T08:28:25,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/b/9118cadcfcf8403ebbb463f433e8468b as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/b/9118cadcfcf8403ebbb463f433e8468b 2024-11-13T08:28:25,378 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/b/9118cadcfcf8403ebbb463f433e8468b, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T08:28:25,380 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/c/da5a4bc078414f2e8bec6a074ecb2944 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/c/da5a4bc078414f2e8bec6a074ecb2944 2024-11-13T08:28:25,386 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/c/da5a4bc078414f2e8bec6a074ecb2944, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T08:28:25,386 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for d0fe7a87c90da5adf1c927ac8b476831 in 73ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T08:28:25,392 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731486500544/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:25,395 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000003002 2024-11-13T08:28:25,395 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T08:28:25,395 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0fe7a87c90da5adf1c927ac8b476831 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T08:28:25,405 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/c/84d5a4f68f644ae7a908383a20b854aa is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731486500511/Put/seqid=0 2024-11-13T08:28:25,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741863_1040 (size=50301) 2024-11-13T08:28:25,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741863_1040 (size=50301) 2024-11-13T08:28:25,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741863_1040 (size=50301) 2024-11-13T08:28:25,415 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/c/84d5a4f68f644ae7a908383a20b854aa 2024-11-13T08:28:25,422 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 84d5a4f68f644ae7a908383a20b854aa 2024-11-13T08:28:25,423 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/.tmp/c/84d5a4f68f644ae7a908383a20b854aa as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/c/84d5a4f68f644ae7a908383a20b854aa 2024-11-13T08:28:25,430 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 84d5a4f68f644ae7a908383a20b854aa 2024-11-13T08:28:25,430 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/c/84d5a4f68f644ae7a908383a20b854aa, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T08:28:25,431 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for d0fe7a87c90da5adf1c927ac8b476831 in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T08:28:25,432 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/0000000000000003002 2024-11-13T08:28:25,433 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:25,433 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:25,434 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T08:28:25,436 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0fe7a87c90da5adf1c927ac8b476831 2024-11-13T08:28:25,438 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenIntoWAL/d0fe7a87c90da5adf1c927ac8b476831/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T08:28:25,439 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0fe7a87c90da5adf1c927ac8b476831; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64518181, jitterRate=-0.03860418498516083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T08:28:25,439 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0fe7a87c90da5adf1c927ac8b476831: Writing region info on filesystem at 1731486504723Initializing all the Stores at 1731486504724 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486504725 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486504725Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486504725Cleaning up temporary data from old regions at 1731486505433 (+708 ms)Region opened successfully at 1731486505439 (+6 ms) 2024-11-13T08:28:25,495 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0fe7a87c90da5adf1c927ac8b476831, disabling compactions & flushes 2024-11-13T08:28:25,495 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:25,495 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:25,495 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. after waiting 0 ms 2024-11-13T08:28:25,495 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:25,497 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731486500311.d0fe7a87c90da5adf1c927ac8b476831. 2024-11-13T08:28:25,497 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0fe7a87c90da5adf1c927ac8b476831: Waiting for close lock at 1731486505494Disabling compacts and flushes for region at 1731486505494Disabling writes for close at 1731486505495 (+1 ms)Writing region close event to WAL at 1731486505497 (+2 ms)Closed at 1731486505497 2024-11-13T08:28:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741857_1034 (size=93) 2024-11-13T08:28:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741857_1034 (size=93) 2024-11-13T08:28:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741857_1034 (size=93) 2024-11-13T08:28:25,504 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:25,504 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731486504688) 2024-11-13T08:28:25,521 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 383) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37607 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:38569 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_478365697_22 at /127.0.0.1:42222 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_478365697_22 at /127.0.0.1:60662 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38569 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33907 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:37607 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_478365697_22 at /127.0.0.1:53842 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_478365697_22 at /127.0.0.1:42266 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=1944 (was 2003) 2024-11-13T08:28:25,535 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=1942 2024-11-13T08:28:25,560 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:25,563 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:25,564 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:25,567 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19433875, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-19433875, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:25,588 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19433875/hregion-19433875.1731486505567, exclude list is [], retry=0 2024-11-13T08:28:25,592 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:25,592 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:25,593 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:25,596 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19433875/hregion-19433875.1731486505567 2024-11-13T08:28:25,596 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:25,596 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ca69817a96fa813897a9a2377ced99cc, NAME => 'test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:25,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741865_1042 (size=43) 2024-11-13T08:28:25,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741865_1042 (size=43) 2024-11-13T08:28:25,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741865_1042 (size=43) 2024-11-13T08:28:25,615 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:25,617 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,618 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca69817a96fa813897a9a2377ced99cc columnFamilyName a 2024-11-13T08:28:25,618 DEBUG [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:25,619 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(327): Store=ca69817a96fa813897a9a2377ced99cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:25,619 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,620 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca69817a96fa813897a9a2377ced99cc columnFamilyName b 2024-11-13T08:28:25,621 DEBUG [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:25,621 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(327): Store=ca69817a96fa813897a9a2377ced99cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:25,621 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,623 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca69817a96fa813897a9a2377ced99cc columnFamilyName c 2024-11-13T08:28:25,623 DEBUG [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:25,623 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(327): Store=ca69817a96fa813897a9a2377ced99cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:25,624 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,624 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,625 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,626 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,626 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,627 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:25,629 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:25,631 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:25,632 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca69817a96fa813897a9a2377ced99cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64962446, jitterRate=-0.0319841206073761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:25,634 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca69817a96fa813897a9a2377ced99cc: Writing region info on filesystem at 1731486505615Initializing all the Stores at 1731486505616 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486505616Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486505616Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486505616Cleaning up temporary data from old regions at 1731486505626 (+10 ms)Region opened successfully at 1731486505634 (+8 ms) 2024-11-13T08:28:25,634 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca69817a96fa813897a9a2377ced99cc, disabling compactions & flushes 2024-11-13T08:28:25,634 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:25,634 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:25,634 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. after waiting 0 ms 2024-11-13T08:28:25,634 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:25,635 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:25,635 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca69817a96fa813897a9a2377ced99cc: Waiting for close lock at 1731486505634Disabling compacts and flushes for region at 1731486505634Disabling writes for close at 1731486505634Writing region close event to WAL at 1731486505635 (+1 ms)Closed at 1731486505635 2024-11-13T08:28:25,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741864_1041 (size=93) 2024-11-13T08:28:25,640 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-19433875/hregion-19433875.1731486505567 not finished, retry = 0 2024-11-13T08:28:25,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741864_1041 (size=93) 2024-11-13T08:28:25,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741864_1041 (size=93) 2024-11-13T08:28:25,743 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:25,744 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19433875:(num 1731486505567) 2024-11-13T08:28:25,744 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:25,746 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:25,760 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747, exclude list is [], retry=0 2024-11-13T08:28:25,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:25,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:25,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:25,766 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 2024-11-13T08:28:25,767 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:25,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741866_1043 (size=344306) 2024-11-13T08:28:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741866_1043 (size=344306) 2024-11-13T08:28:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741866_1043 (size=344306) 2024-11-13T08:28:25,919 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747, size=336.2 K (344306bytes) 2024-11-13T08:28:25,919 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 2024-11-13T08:28:25,919 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 after 0ms 2024-11-13T08:28:25,924 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 took 6ms 2024-11-13T08:28:25,929 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731486505747.temp 2024-11-13T08:28:25,931 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000000001-wal.1731486505747.temp 2024-11-13T08:28:25,951 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 so closing down 2024-11-13T08:28:25,951 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:25,951 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:25,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741867_1044 (size=344306) 2024-11-13T08:28:25,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741867_1044 (size=344306) 2024-11-13T08:28:25,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741867_1044 (size=344306) 2024-11-13T08:28:25,961 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000000001-wal.1731486505747.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-13T08:28:25,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000000001-wal.1731486505747.temp to hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003000 2024-11-13T08:28:25,963 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-13T08:28:25,963 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747, journal: Splitting hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747, size=336.2 K (344306bytes) at 1731486505919Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000000001-wal.1731486505747.temp at 1731486505931 (+12 ms)Split 1024 edits, skipped 0 edits. at 1731486505934 (+3 ms)Split 2048 edits, skipped 0 edits. at 1731486505943 (+9 ms)Finishing writing output for hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 so closing down at 1731486505951 (+8 ms)3 split writer threads finished at 1731486505951Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000000001-wal.1731486505747.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1731486505961 (+10 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000000001-wal.1731486505747.temp to hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003000 at 1731486505963 (+2 ms)Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731486505963 2024-11-13T08:28:25,965 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505747 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486505747 2024-11-13T08:28:25,966 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003000 2024-11-13T08:28:25,966 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:25,968 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:25,983 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969, exclude list is [], retry=0 2024-11-13T08:28:25,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:25,987 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:25,987 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:25,989 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 2024-11-13T08:28:25,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:26,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741868_1045 (size=344433) 2024-11-13T08:28:26,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741868_1045 (size=344433) 2024-11-13T08:28:26,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741868_1045 (size=344433) 2024-11-13T08:28:26,204 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969, size=336.4 K (344433bytes) 2024-11-13T08:28:26,204 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 2024-11-13T08:28:26,205 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 after 1ms 2024-11-13T08:28:26,209 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 took 5ms 2024-11-13T08:28:26,214 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731486505969.temp 2024-11-13T08:28:26,215 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003001-wal.1731486505969.temp 2024-11-13T08:28:26,248 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 so closing down 2024-11-13T08:28:26,248 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:26,249 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:26,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741869_1046 (size=344433) 2024-11-13T08:28:26,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741869_1046 (size=344433) 2024-11-13T08:28:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741869_1046 (size=344433) 2024-11-13T08:28:26,254 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003001-wal.1731486505969.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-13T08:28:26,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003001-wal.1731486505969.temp to hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000006000 2024-11-13T08:28:26,256 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 46 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-13T08:28:26,256 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969, journal: Splitting hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969, size=336.4 K (344433bytes) at 1731486506204Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003001-wal.1731486505969.temp at 1731486506215 (+11 ms)Split 1024 edits, skipped 0 edits. at 1731486506221 (+6 ms)Split 2048 edits, skipped 0 edits. at 1731486506235 (+14 ms)Finishing writing output for hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 so closing down at 1731486506248 (+13 ms)3 split writer threads finished at 1731486506249 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003001-wal.1731486505969.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1731486506254 (+5 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003001-wal.1731486505969.temp to hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000006000 at 1731486506256 (+2 ms)Processed 3000 edits across 1 Regions in 46 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731486506256 2024-11-13T08:28:26,258 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486505969 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486505969 2024-11-13T08:28:26,260 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000006000 2024-11-13T08:28:26,260 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:26,262 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/test2727-manual,16010,1731486505559, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:26,285 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486506263, exclude list is [], retry=0 2024-11-13T08:28:26,289 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:26,290 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:26,290 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:26,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731486505559/wal.1731486506263 2024-11-13T08:28:26,293 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:26,293 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca69817a96fa813897a9a2377ced99cc, NAME => 'test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:26,293 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:26,293 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,293 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,296 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,297 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca69817a96fa813897a9a2377ced99cc columnFamilyName a 2024-11-13T08:28:26,297 DEBUG [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,298 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(327): Store=ca69817a96fa813897a9a2377ced99cc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,298 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,299 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca69817a96fa813897a9a2377ced99cc columnFamilyName b 2024-11-13T08:28:26,299 DEBUG [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,300 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(327): Store=ca69817a96fa813897a9a2377ced99cc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,300 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,302 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca69817a96fa813897a9a2377ced99cc columnFamilyName c 2024-11-13T08:28:26,302 DEBUG [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,302 INFO [StoreOpener-ca69817a96fa813897a9a2377ced99cc-1 {}] regionserver.HStore(327): Store=ca69817a96fa813897a9a2377ced99cc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,302 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,303 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,307 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003000 2024-11-13T08:28:26,353 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003000 2024-11-13T08:28:26,355 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000006000 2024-11-13T08:28:26,398 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000006000 2024-11-13T08:28:26,398 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca69817a96fa813897a9a2377ced99cc 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T08:28:26,423 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/a/1b4180f0a149422f8956eea4c5d37bf1 is 41, key is test2727/a:100/1731486505995/Put/seqid=0 2024-11-13T08:28:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741871_1048 (size=84227) 2024-11-13T08:28:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741871_1048 (size=84227) 2024-11-13T08:28:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741871_1048 (size=84227) 2024-11-13T08:28:26,432 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/a/1b4180f0a149422f8956eea4c5d37bf1 2024-11-13T08:28:26,462 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/b/b926983de9534c61997ad52a1cb740e5 is 41, key is test2727/b:100/1731486506063/Put/seqid=0 2024-11-13T08:28:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741872_1049 (size=84609) 2024-11-13T08:28:26,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741872_1049 (size=84609) 2024-11-13T08:28:26,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741872_1049 (size=84609) 2024-11-13T08:28:26,470 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/b/b926983de9534c61997ad52a1cb740e5 2024-11-13T08:28:26,498 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/c/6833a8eac64a45f88f20c21654a70263 is 41, key is test2727/c:100/1731486506128/Put/seqid=0 2024-11-13T08:28:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741873_1050 (size=84609) 2024-11-13T08:28:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741873_1050 (size=84609) 2024-11-13T08:28:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741873_1050 (size=84609) 2024-11-13T08:28:26,509 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/c/6833a8eac64a45f88f20c21654a70263 2024-11-13T08:28:26,518 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/a/1b4180f0a149422f8956eea4c5d37bf1 as hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/a/1b4180f0a149422f8956eea4c5d37bf1 2024-11-13T08:28:26,525 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/a/1b4180f0a149422f8956eea4c5d37bf1, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T08:28:26,526 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/b/b926983de9534c61997ad52a1cb740e5 as hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/b/b926983de9534c61997ad52a1cb740e5 2024-11-13T08:28:26,533 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/b/b926983de9534c61997ad52a1cb740e5, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T08:28:26,535 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/.tmp/c/6833a8eac64a45f88f20c21654a70263 as hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/c/6833a8eac64a45f88f20c21654a70263 2024-11-13T08:28:26,542 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/c/6833a8eac64a45f88f20c21654a70263, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T08:28:26,542 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ca69817a96fa813897a9a2377ced99cc in 144ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T08:28:26,543 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000003000 2024-11-13T08:28:26,543 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/0000000000000006000 2024-11-13T08:28:26,545 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,545 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,545 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:26,547 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca69817a96fa813897a9a2377ced99cc 2024-11-13T08:28:26,550 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/test2727/ca69817a96fa813897a9a2377ced99cc/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T08:28:26,551 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca69817a96fa813897a9a2377ced99cc; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73090059, jitterRate=0.08912675082683563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:26,553 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca69817a96fa813897a9a2377ced99cc: Writing region info on filesystem at 1731486506294Initializing all the Stores at 1731486506295 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506295Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506296 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506296Obtaining lock to block concurrent updates at 1731486506398 (+102 ms)Preparing flush snapshotting stores in ca69817a96fa813897a9a2377ced99cc at 1731486506398Finished memstore snapshotting test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731486506399 (+1 ms)Flushing stores of test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. at 1731486506399Flushing ca69817a96fa813897a9a2377ced99cc/a: creating writer at 1731486506399Flushing ca69817a96fa813897a9a2377ced99cc/a: appending metadata at 1731486506422 (+23 ms)Flushing ca69817a96fa813897a9a2377ced99cc/a: closing flushed file at 1731486506422Flushing ca69817a96fa813897a9a2377ced99cc/b: creating writer at 1731486506439 (+17 ms)Flushing ca69817a96fa813897a9a2377ced99cc/b: appending metadata at 1731486506460 (+21 ms)Flushing ca69817a96fa813897a9a2377ced99cc/b: closing flushed file at 1731486506461 (+1 ms)Flushing ca69817a96fa813897a9a2377ced99cc/c: creating writer at 1731486506477 (+16 ms)Flushing ca69817a96fa813897a9a2377ced99cc/c: appending metadata at 1731486506497 (+20 ms)Flushing ca69817a96fa813897a9a2377ced99cc/c: closing flushed file at 1731486506497Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30a7e25a: reopening flushed file at 1731486506517 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26a4a25f: reopening flushed file at 1731486506525 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34fb1b6f: reopening flushed file at 1731486506534 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ca69817a96fa813897a9a2377ced99cc in 144ms, sequenceid=6000, compaction requested=false; wal=null at 1731486506542 (+8 ms)Cleaning up temporary data from old regions at 1731486506545 (+3 ms)Region opened successfully at 1731486506553 (+8 ms) 2024-11-13T08:28:26,554 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T08:28:26,554 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca69817a96fa813897a9a2377ced99cc, disabling compactions & flushes 2024-11-13T08:28:26,554 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:26,554 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:26,554 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. after waiting 0 ms 2024-11-13T08:28:26,554 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:26,555 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731486505561.ca69817a96fa813897a9a2377ced99cc. 2024-11-13T08:28:26,556 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca69817a96fa813897a9a2377ced99cc: Waiting for close lock at 1731486506554Disabling compacts and flushes for region at 1731486506554Disabling writes for close at 1731486506554Writing region close event to WAL at 1731486506555 (+1 ms)Closed at 1731486506555 2024-11-13T08:28:26,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741870_1047 (size=93) 2024-11-13T08:28:26,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741870_1047 (size=93) 2024-11-13T08:28:26,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741870_1047 (size=93) 2024-11-13T08:28:26,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:26,562 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731486506263) 2024-11-13T08:28:26,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=402 (was 399) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:60840 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53934 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=1837 (was 1942) 2024-11-13T08:28:26,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=1836 2024-11-13T08:28:26,607 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:26,614 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:26,615 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731486506615 2024-11-13T08:28:26,623 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615 2024-11-13T08:28:26,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:26,626 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ace99940cd2aba1c2fe0ae6d34d092f, NAME => 'testSequentialEditLogSeqNum,,1731486506608.6ace99940cd2aba1c2fe0ae6d34d092f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:26,626 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731486506608.6ace99940cd2aba1c2fe0ae6d34d092f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:26,626 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,626 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,627 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f doesn't exist for region: 6ace99940cd2aba1c2fe0ae6d34d092f on table testSequentialEditLogSeqNum 2024-11-13T08:28:26,628 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 6ace99940cd2aba1c2fe0ae6d34d092f on table testSequentialEditLogSeqNum 2024-11-13T08:28:26,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741875_1052 (size=62) 2024-11-13T08:28:26,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741875_1052 (size=62) 2024-11-13T08:28:26,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741875_1052 (size=62) 2024-11-13T08:28:26,639 INFO [StoreOpener-6ace99940cd2aba1c2fe0ae6d34d092f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,641 INFO [StoreOpener-6ace99940cd2aba1c2fe0ae6d34d092f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ace99940cd2aba1c2fe0ae6d34d092f columnFamilyName a 2024-11-13T08:28:26,641 DEBUG [StoreOpener-6ace99940cd2aba1c2fe0ae6d34d092f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,641 INFO [StoreOpener-6ace99940cd2aba1c2fe0ae6d34d092f-1 {}] regionserver.HStore(327): Store=6ace99940cd2aba1c2fe0ae6d34d092f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,641 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,643 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,643 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,645 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6ace99940cd2aba1c2fe0ae6d34d092f 2024-11-13T08:28:26,647 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:26,648 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6ace99940cd2aba1c2fe0ae6d34d092f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60246984, jitterRate=-0.10224997997283936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:26,649 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6ace99940cd2aba1c2fe0ae6d34d092f: Writing region info on filesystem at 1731486506627Initializing all the Stores at 1731486506639 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506639Cleaning up temporary data from old regions at 1731486506643 (+4 ms)Region opened successfully at 1731486506649 (+6 ms) 2024-11-13T08:28:26,661 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6ace99940cd2aba1c2fe0ae6d34d092f 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T08:28:26,692 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/.tmp/a/9057d7f31f1f4d9993e49629eb152b57 is 81, key is testSequentialEditLogSeqNum/a:x0/1731486506649/Put/seqid=0 2024-11-13T08:28:26,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741876_1053 (size=5833) 2024-11-13T08:28:26,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741876_1053 (size=5833) 2024-11-13T08:28:26,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741876_1053 (size=5833) 2024-11-13T08:28:26,701 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/.tmp/a/9057d7f31f1f4d9993e49629eb152b57 2024-11-13T08:28:26,709 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/.tmp/a/9057d7f31f1f4d9993e49629eb152b57 as hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/a/9057d7f31f1f4d9993e49629eb152b57 2024-11-13T08:28:26,717 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/a/9057d7f31f1f4d9993e49629eb152b57, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T08:28:26,718 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6ace99940cd2aba1c2fe0ae6d34d092f in 57ms, sequenceid=13, compaction requested=false 2024-11-13T08:28:26,719 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6ace99940cd2aba1c2fe0ae6d34d092f: 2024-11-13T08:28:26,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T08:28:26,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T08:28:26,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T08:28:26,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T08:28:26,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T08:28:26,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741874_1051 (size=3017) 2024-11-13T08:28:26,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741874_1051 (size=3017) 2024-11-13T08:28:26,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741874_1051 (size=3017) 2024-11-13T08:28:26,746 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615, size=2.9 K (3017bytes) 2024-11-13T08:28:26,746 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615 2024-11-13T08:28:26,747 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615 after 1ms 2024-11-13T08:28:26,749 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615 took 3ms 2024-11-13T08:28:26,752 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615 so closing down 2024-11-13T08:28:26,752 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:26,753 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731486506615.temp 2024-11-13T08:28:26,754 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000003-wal.1731486506615.temp 2024-11-13T08:28:26,754 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:26,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741877_1054 (size=2433) 2024-11-13T08:28:26,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741877_1054 (size=2433) 2024-11-13T08:28:26,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741877_1054 (size=2433) 2024-11-13T08:28:26,763 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000003-wal.1731486506615.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:26,765 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000003-wal.1731486506615.temp to hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000020 2024-11-13T08:28:26,765 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-13T08:28:26,765 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615, journal: Splitting hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615, size=2.9 K (3017bytes) at 1731486506746Finishing writing output for hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615 so closing down at 1731486506752 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000003-wal.1731486506615.temp at 1731486506754 (+2 ms)3 split writer threads finished at 1731486506755 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000003-wal.1731486506615.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731486506763 (+8 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000003-wal.1731486506615.temp to hdfs://localhost:38569/hbase/data/default/testSequentialEditLogSeqNum/6ace99940cd2aba1c2fe0ae6d34d092f/recovered.edits/0000000000000000020 at 1731486506765 (+2 ms)Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38569/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731486506606/wal.1731486506615, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731486506765 2024-11-13T08:28:26,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:60840 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:53934 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=1812 (was 1836) 2024-11-13T08:28:26,796 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=1808 2024-11-13T08:28:26,814 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:26,817 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:26,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:26,852 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68590046, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-68590046, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:26,875 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68590046/hregion-68590046.1731486506853, exclude list is [], retry=0 2024-11-13T08:28:26,879 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:26,879 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:26,879 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:26,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68590046/hregion-68590046.1731486506853 2024-11-13T08:28:26,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:26,885 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a74fdca2d4c32f9a410a23f029c15023, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:26,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741879_1056 (size=70) 2024-11-13T08:28:26,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741879_1056 (size=70) 2024-11-13T08:28:26,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741879_1056 (size=70) 2024-11-13T08:28:26,896 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:26,898 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,900 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName a 2024-11-13T08:28:26,900 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,901 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,901 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,903 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName b 2024-11-13T08:28:26,903 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,903 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,903 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,905 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName c 2024-11-13T08:28:26,905 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,905 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,906 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,907 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,908 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,908 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,909 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:26,910 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,913 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:26,913 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a74fdca2d4c32f9a410a23f029c15023; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61027730, jitterRate=-0.09061595797538757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:26,915 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a74fdca2d4c32f9a410a23f029c15023: Writing region info on filesystem at 1731486506896Initializing all the Stores at 1731486506898 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506898Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506898Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506898Cleaning up temporary data from old regions at 1731486506908 (+10 ms)Region opened successfully at 1731486506914 (+6 ms) 2024-11-13T08:28:26,915 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a74fdca2d4c32f9a410a23f029c15023, disabling compactions & flushes 2024-11-13T08:28:26,915 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:26,915 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:26,915 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. after waiting 0 ms 2024-11-13T08:28:26,915 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:26,916 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:26,916 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a74fdca2d4c32f9a410a23f029c15023: Waiting for close lock at 1731486506915Disabling compacts and flushes for region at 1731486506915Disabling writes for close at 1731486506915Writing region close event to WAL at 1731486506916 (+1 ms)Closed at 1731486506916 2024-11-13T08:28:26,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741878_1055 (size=93) 2024-11-13T08:28:26,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741878_1055 (size=93) 2024-11-13T08:28:26,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741878_1055 (size=93) 2024-11-13T08:28:26,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:26,923 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68590046:(num 1731486506853) 2024-11-13T08:28:26,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:26,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:26,947 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926, exclude list is [], retry=0 2024-11-13T08:28:26,951 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:26,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:26,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:26,954 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 2024-11-13T08:28:26,955 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:26,955 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a74fdca2d4c32f9a410a23f029c15023, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:26,955 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:26,955 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,955 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,957 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,959 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName a 2024-11-13T08:28:26,959 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,960 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,960 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,961 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName b 2024-11-13T08:28:26,961 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,962 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,962 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,963 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName c 2024-11-13T08:28:26,963 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:26,963 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:26,964 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,964 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,967 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,967 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,968 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:26,970 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:26,971 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a74fdca2d4c32f9a410a23f029c15023; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73120880, jitterRate=0.08958601951599121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:26,973 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a74fdca2d4c32f9a410a23f029c15023: Writing region info on filesystem at 1731486506956Initializing all the Stores at 1731486506957 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506957Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506957Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486506957Cleaning up temporary data from old regions at 1731486506967 (+10 ms)Region opened successfully at 1731486506973 (+6 ms) 2024-11-13T08:28:26,979 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731486506977/Put/seqid=0 2024-11-13T08:28:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741881_1058 (size=4826) 2024-11-13T08:28:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741881_1058 (size=4826) 2024-11-13T08:28:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741881_1058 (size=4826) 2024-11-13T08:28:26,987 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38569/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in a74fdca2d4c32f9a410a23f029c15023/a 2024-11-13T08:28:26,997 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T08:28:26,997 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T08:28:26,998 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a74fdca2d4c32f9a410a23f029c15023: 2024-11-13T08:28:27,000 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/64c6e2788a144176bf91a9cd23854ff1_SeqId_3_ 2024-11-13T08:28:27,001 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38569/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a74fdca2d4c32f9a410a23f029c15023/a as hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/64c6e2788a144176bf91a9cd23854ff1_SeqId_3_ - updating store file list. 2024-11-13T08:28:27,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T08:28:27,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:27,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T08:28:27,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:27,009 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 64c6e2788a144176bf91a9cd23854ff1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:27,009 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/64c6e2788a144176bf91a9cd23854ff1_SeqId_3_ into a74fdca2d4c32f9a410a23f029c15023/a 2024-11-13T08:28:27,009 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38569/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a74fdca2d4c32f9a410a23f029c15023/a (new location: hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/64c6e2788a144176bf91a9cd23854ff1_SeqId_3_) 2024-11-13T08:28:27,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T08:28:27,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T08:28:27,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T08:28:27,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:27,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T08:28:27,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:27,012 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T08:28:27,012 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:27,062 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926, size=0 (0bytes) 2024-11-13T08:28:27,062 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 might be still open, length is 0 2024-11-13T08:28:27,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 2024-11-13T08:28:27,063 WARN [IPC Server handler 2 on default port 38569 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T08:28:27,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 after 1ms 2024-11-13T08:28:27,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:54028 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44221:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54028 dst: /127.0.0.1:44221 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44221 remote=/127.0.0.1:54028]. Total timeout mills is 60000, 59494 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:27,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:60946 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34057:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60946 dst: /127.0.0.1:34057 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:27,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:42422 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42422 dst: /127.0.0.1:34115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:27,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741880_1059 (size=504) 2024-11-13T08:28:27,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741880_1059 (size=504) 2024-11-13T08:28:27,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741880_1059 (size=504) 2024-11-13T08:28:31,064 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 after 4002ms 2024-11-13T08:28:31,068 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 took 4006ms 2024-11-13T08:28:31,070 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926; continuing. 2024-11-13T08:28:31,070 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 so closing down 2024-11-13T08:28:31,070 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:31,072 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731486506926.temp 2024-11-13T08:28:31,073 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005-wal.1731486506926.temp 2024-11-13T08:28:31,073 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:31,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741882_1060 (size=238) 2024-11-13T08:28:31,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741882_1060 (size=238) 2024-11-13T08:28:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741882_1060 (size=238) 2024-11-13T08:28:31,082 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005-wal.1731486506926.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:31,084 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005-wal.1731486506926.temp to hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005 2024-11-13T08:28:31,084 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T08:28:31,084 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926, journal: Splitting hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926, size=0 (0bytes) at 1731486507062Finishing writing output for hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 so closing down at 1731486511070 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005-wal.1731486506926.temp at 1731486511073 (+3 ms)3 split writer threads finished at 1731486511073Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005-wal.1731486506926.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731486511082 (+9 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005-wal.1731486506926.temp to hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005 at 1731486511084 (+2 ms)Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926, size=0, length=0, corrupted=false, cancelled=false at 1731486511084 2024-11-13T08:28:31,086 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486506926 2024-11-13T08:28:31,086 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005 2024-11-13T08:28:31,087 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:31,088 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:31,104 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486511089, exclude list is [], retry=0 2024-11-13T08:28:31,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:31,107 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:31,107 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:31,109 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486511089 2024-11-13T08:28:31,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:31,109 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a74fdca2d4c32f9a410a23f029c15023, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:31,109 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:31,109 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,110 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,111 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,112 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName a 2024-11-13T08:28:31,112 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,118 DEBUG [StoreFileOpener-a74fdca2d4c32f9a410a23f029c15023-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 64c6e2788a144176bf91a9cd23854ff1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:31,119 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/64c6e2788a144176bf91a9cd23854ff1_SeqId_3_ 2024-11-13T08:28:31,119 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,119 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,120 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName b 2024-11-13T08:28:31,120 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,120 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,121 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,122 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a74fdca2d4c32f9a410a23f029c15023 columnFamilyName c 2024-11-13T08:28:31,122 DEBUG [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,122 INFO [StoreOpener-a74fdca2d4c32f9a410a23f029c15023-1 {}] regionserver.HStore(327): Store=a74fdca2d4c32f9a410a23f029c15023/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,122 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,125 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,126 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005 2024-11-13T08:28:31,128 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005 2024-11-13T08:28:31,129 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a74fdca2d4c32f9a410a23f029c15023 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T08:28:31,145 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/.tmp/a/f94430ff2eba43b2ad987369b8c399fc is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731486507016/Put/seqid=0 2024-11-13T08:28:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741884_1062 (size=5149) 2024-11-13T08:28:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741884_1062 (size=5149) 2024-11-13T08:28:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741884_1062 (size=5149) 2024-11-13T08:28:31,153 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/.tmp/a/f94430ff2eba43b2ad987369b8c399fc 2024-11-13T08:28:31,159 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/.tmp/a/f94430ff2eba43b2ad987369b8c399fc as hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/f94430ff2eba43b2ad987369b8c399fc 2024-11-13T08:28:31,165 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/a/f94430ff2eba43b2ad987369b8c399fc, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T08:28:31,165 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a74fdca2d4c32f9a410a23f029c15023 in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T08:28:31,166 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/0000000000000000005 2024-11-13T08:28:31,167 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,167 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,167 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:31,169 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a74fdca2d4c32f9a410a23f029c15023 2024-11-13T08:28:31,171 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a74fdca2d4c32f9a410a23f029c15023/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T08:28:31,172 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a74fdca2d4c32f9a410a23f029c15023; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73585457, jitterRate=0.09650875627994537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:31,172 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a74fdca2d4c32f9a410a23f029c15023: Writing region info on filesystem at 1731486511110Initializing all the Stores at 1731486511111 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486511111Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486511111Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486511111Obtaining lock to block concurrent updates at 1731486511129 (+18 ms)Preparing flush snapshotting stores in a74fdca2d4c32f9a410a23f029c15023 at 1731486511129Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731486511129Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. at 1731486511129Flushing a74fdca2d4c32f9a410a23f029c15023/a: creating writer at 1731486511129Flushing a74fdca2d4c32f9a410a23f029c15023/a: appending metadata at 1731486511145 (+16 ms)Flushing a74fdca2d4c32f9a410a23f029c15023/a: closing flushed file at 1731486511145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662dc92d: reopening flushed file at 1731486511158 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a74fdca2d4c32f9a410a23f029c15023 in 37ms, sequenceid=5, compaction requested=false; wal=null at 1731486511165 (+7 ms)Cleaning up temporary data from old regions at 1731486511167 (+2 ms)Region opened successfully at 1731486511172 (+5 ms) 2024-11-13T08:28:31,176 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a74fdca2d4c32f9a410a23f029c15023, disabling compactions & flushes 2024-11-13T08:28:31,176 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:31,176 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:31,176 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. after waiting 0 ms 2024-11-13T08:28:31,176 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:31,177 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731486506815.a74fdca2d4c32f9a410a23f029c15023. 2024-11-13T08:28:31,177 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a74fdca2d4c32f9a410a23f029c15023: Waiting for close lock at 1731486511176Disabling compacts and flushes for region at 1731486511176Disabling writes for close at 1731486511176Writing region close event to WAL at 1731486511177 (+1 ms)Closed at 1731486511177 2024-11-13T08:28:31,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741883_1061 (size=93) 2024-11-13T08:28:31,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741883_1061 (size=93) 2024-11-13T08:28:31,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741883_1061 (size=93) 2024-11-13T08:28:31,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:31,182 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731486511089) 2024-11-13T08:28:31,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408 (was 407) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1206919010_22 at /127.0.0.1:55992 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1206919010_22 at /127.0.0.1:56614 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38569 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:38569 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1206919010_22 at /127.0.0.1:43222 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=146 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=1758 (was 1808) 2024-11-13T08:28:31,197 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-13T08:28:31,209 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=146, ProcessCount=11, AvailableMemoryMB=1757 2024-11-13T08:28:31,210 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-13T08:28:31,227 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:31,232 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T08:28:31,237 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:31,240 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7febadd 2024-11-13T08:28:31,241 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T08:28:31,243 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T08:28:31,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T08:28:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T08:28:31,256 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T08:28:31,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T08:28:31,258 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T08:28:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T08:28:31,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741885_1063 (size=694) 2024-11-13T08:28:31,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741885_1063 (size=694) 2024-11-13T08:28:31,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741885_1063 (size=694) 2024-11-13T08:28:31,276 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc 2024-11-13T08:28:31,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741886_1064 (size=77) 2024-11-13T08:28:31,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741886_1064 (size=77) 2024-11-13T08:28:31,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741886_1064 (size=77) 2024-11-13T08:28:31,286 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:31,286 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:31,286 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,287 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,287 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:31,287 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,287 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,287 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486511286Disabling compacts and flushes for region at 1731486511286Disabling writes for close at 1731486511287 (+1 ms)Writing region close event to WAL at 1731486511287Closed at 1731486511287 2024-11-13T08:28:31,288 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T08:28:31,293 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731486511288"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731486511288"}]},"ts":"1731486511288"} 2024-11-13T08:28:31,297 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T08:28:31,299 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T08:28:31,302 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731486511299"}]},"ts":"1731486511299"} 2024-11-13T08:28:31,310 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T08:28:31,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dd3a0865bb23=0} racks are {/default-rack=0} 2024-11-13T08:28:31,312 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T08:28:31,313 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T08:28:31,313 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T08:28:31,313 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T08:28:31,313 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T08:28:31,313 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T08:28:31,313 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T08:28:31,313 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T08:28:31,313 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T08:28:31,313 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T08:28:31,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN}] 2024-11-13T08:28:31,316 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN 2024-11-13T08:28:31,318 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN; state=OFFLINE, location=dd3a0865bb23,44073,1731486497392; forceNewPlan=false, retain=false 2024-11-13T08:28:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T08:28:31,471 INFO [dd3a0865bb23:37177 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T08:28:31,472 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPENING, regionLocation=dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:31,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN because future has completed 2024-11-13T08:28:31,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392}] 2024-11-13T08:28:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T08:28:31,630 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T08:28:31,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37049, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T08:28:31,638 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,638 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:31,639 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,639 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:31,639 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,639 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,640 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,642 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf1 2024-11-13T08:28:31,642 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,643 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,643 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,644 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf2 2024-11-13T08:28:31,644 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,645 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,645 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,646 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,646 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,647 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,647 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,648 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T08:28:31,649 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,653 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:31,654 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fb116209b3b52bde0867fb4d45873206; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71436364, jitterRate=0.06448477506637573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T08:28:31,654 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,655 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fb116209b3b52bde0867fb4d45873206: Running coprocessor pre-open hook at 1731486511639Writing region info on filesystem at 1731486511639Initializing all the Stores at 1731486511640 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486511640Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486511640Cleaning up temporary data from old regions at 1731486511647 (+7 ms)Running coprocessor post-open hooks at 1731486511654 (+7 ms)Region opened successfully at 1731486511655 (+1 ms) 2024-11-13T08:28:31,657 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., pid=6, masterSystemTime=1731486511630 2024-11-13T08:28:31,660 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,660 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,661 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPEN, openSeqNum=2, regionLocation=dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:31,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392 because future has completed 2024-11-13T08:28:31,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T08:28:31,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392 in 189 msec 2024-11-13T08:28:31,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T08:28:31,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN in 355 msec 2024-11-13T08:28:31,674 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T08:28:31,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731486511674"}]},"ts":"1731486511674"} 2024-11-13T08:28:31,677 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T08:28:31,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T08:28:31,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 430 msec 2024-11-13T08:28:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T08:28:31,895 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T08:28:31,895 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T08:28:31,896 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T08:28:31,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T08:28:31,902 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T08:28:31,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T08:28:31,917 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=2] 2024-11-13T08:28:31,919 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T08:28:31,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T08:28:31,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=fb116209b3b52bde0867fb4d45873206, source=dd3a0865bb23,44073,1731486497392, destination=dd3a0865bb23,35727,1731486497292, warming up region on dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T08:28:31,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=fb116209b3b52bde0867fb4d45873206, source=dd3a0865bb23,44073,1731486497392, destination=dd3a0865bb23,35727,1731486497292, running balancer 2024-11-13T08:28:31,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58623, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T08:28:31,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE 2024-11-13T08:28:31,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE 2024-11-13T08:28:31,946 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=CLOSING, regionLocation=dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:31,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(7855): Warmup {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:31,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE because future has completed 2024-11-13T08:28:31,949 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,950 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T08:28:31,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392}] 2024-11-13T08:28:31,951 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf1 2024-11-13T08:28:31,951 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,952 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,952 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:31,954 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf2 2024-11-13T08:28:31,954 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:31,955 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:31,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486511955Disabling compacts and flushes for region at 1731486511955Disabling writes for close at 1731486511955Writing region close event to WAL at 1731486511956 (+1 ms)Closed at 1731486511956 2024-11-13T08:28:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T08:28:32,112 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,112 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T08:28:32,113 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:32,113 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,113 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,113 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:32,113 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,113 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing fb116209b3b52bde0867fb4d45873206 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T08:28:32,134 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/ec42fb5893064a8a979ba20bab66d0a1 is 35, key is r1/cf1:q/1731486511922/Put/seqid=0 2024-11-13T08:28:32,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741887_1065 (size=4783) 2024-11-13T08:28:32,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741887_1065 (size=4783) 2024-11-13T08:28:32,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741887_1065 (size=4783) 2024-11-13T08:28:32,144 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/ec42fb5893064a8a979ba20bab66d0a1 2024-11-13T08:28:32,153 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/ec42fb5893064a8a979ba20bab66d0a1 as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1 2024-11-13T08:28:32,161 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T08:28:32,162 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for fb116209b3b52bde0867fb4d45873206 in 49ms, sequenceid=5, compaction requested=false 2024-11-13T08:28:32,162 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T08:28:32,169 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T08:28:32,172 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,172 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486512113Running coprocessor pre-close hooks at 1731486512113Disabling compacts and flushes for region at 1731486512113Disabling writes for close at 1731486512113Obtaining lock to block concurrent updates at 1731486512113Preparing flush snapshotting stores in fb116209b3b52bde0867fb4d45873206 at 1731486512113Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731486512113Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. at 1731486512114 (+1 ms)Flushing fb116209b3b52bde0867fb4d45873206/cf1: creating writer at 1731486512114Flushing fb116209b3b52bde0867fb4d45873206/cf1: appending metadata at 1731486512134 (+20 ms)Flushing fb116209b3b52bde0867fb4d45873206/cf1: closing flushed file at 1731486512134Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ff54e1: reopening flushed file at 1731486512152 (+18 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for fb116209b3b52bde0867fb4d45873206 in 49ms, sequenceid=5, compaction requested=false at 1731486512162 (+10 ms)Writing region close event to WAL at 1731486512164 (+2 ms)Running coprocessor post-close hooks at 1731486512170 (+6 ms)Closed at 1731486512172 (+2 ms) 2024-11-13T08:28:32,173 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding fb116209b3b52bde0867fb4d45873206 move to dd3a0865bb23,35727,1731486497292 record at close sequenceid=5 2024-11-13T08:28:32,176 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,177 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=CLOSED 2024-11-13T08:28:32,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392 because future has completed 2024-11-13T08:28:32,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T08:28:32,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392 in 233 msec 2024-11-13T08:28:32,187 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE; state=CLOSED, location=dd3a0865bb23,35727,1731486497292; forceNewPlan=false, retain=false 2024-11-13T08:28:32,337 INFO [dd3a0865bb23:37177 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T08:28:32,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPENING, regionLocation=dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:32,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE because future has completed 2024-11-13T08:28:32,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292}] 2024-11-13T08:28:32,498 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,498 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:32,498 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,499 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:32,499 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,499 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,500 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,501 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf1 2024-11-13T08:28:32,501 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:32,508 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1 2024-11-13T08:28:32,508 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:32,508 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,509 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf2 2024-11-13T08:28:32,509 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:32,509 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:32,510 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,510 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,512 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,512 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,513 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,513 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T08:28:32,515 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,516 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened fb116209b3b52bde0867fb4d45873206; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74232011, jitterRate=0.10614316165447235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T08:28:32,516 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,517 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for fb116209b3b52bde0867fb4d45873206: Running coprocessor pre-open hook at 1731486512499Writing region info on filesystem at 1731486512499Initializing all the Stores at 1731486512500 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486512500Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486512500Cleaning up temporary data from old regions at 1731486512513 (+13 ms)Running coprocessor post-open hooks at 1731486512516 (+3 ms)Region opened successfully at 1731486512517 (+1 ms) 2024-11-13T08:28:32,518 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., pid=9, masterSystemTime=1731486512493 2024-11-13T08:28:32,521 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,521 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPEN, openSeqNum=9, regionLocation=dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:32,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292 because future has completed 2024-11-13T08:28:32,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T08:28:32,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292 in 185 msec 2024-11-13T08:28:32,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE in 587 msec 2024-11-13T08:28:32,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T08:28:32,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T08:28:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:41800 deadline: 1731486572583, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=35727 startCode=1731486497292. As of locationSeqNum=5. 2024-11-13T08:28:32,590 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=35727 startCode=1731486497292. As of locationSeqNum=5. 2024-11-13T08:28:32,591 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=35727 startCode=1731486497292. As of locationSeqNum=5. 2024-11-13T08:28:32,591 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,35727,1731486497292, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=35727 startCode=1731486497292. As of locationSeqNum=5. 2024-11-13T08:28:32,703 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T08:28:32,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T08:28:32,715 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fb116209b3b52bde0867fb4d45873206 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T08:28:32,733 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/8960a6bc4db048919d1fd9eb4617bccf is 29, key is r1/cf1:/1731486512706/DeleteFamily/seqid=0 2024-11-13T08:28:32,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741888_1066 (size=4906) 2024-11-13T08:28:32,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741888_1066 (size=4906) 2024-11-13T08:28:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741888_1066 (size=4906) 2024-11-13T08:28:32,741 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:32,748 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:32,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf2/e673242623ca4582ac9009c28defe46b is 29, key is r1/cf2:/1731486512706/DeleteFamily/seqid=0 2024-11-13T08:28:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741889_1067 (size=4906) 2024-11-13T08:28:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741889_1067 (size=4906) 2024-11-13T08:28:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741889_1067 (size=4906) 2024-11-13T08:28:32,771 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf2/e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:32,777 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:32,778 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/8960a6bc4db048919d1fd9eb4617bccf as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:32,784 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:32,784 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T08:28:32,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf2/e673242623ca4582ac9009c28defe46b as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:32,790 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:32,791 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T08:28:32,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for fb116209b3b52bde0867fb4d45873206 in 77ms, sequenceid=12, compaction requested=false 2024-11-13T08:28:32,792 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fb116209b3b52bde0867fb4d45873206: 2024-11-13T08:28:32,794 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T08:28:32,796 DEBUG [Time-limited test {}] regionserver.HStore(1541): fb116209b3b52bde0867fb4d45873206/cf1 is initiating major compaction (all files) 2024-11-13T08:28:32,796 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T08:28:32,796 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:32,797 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of fb116209b3b52bde0867fb4d45873206/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,797 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1, hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf] into tmpdir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp, totalSize=9.5 K 2024-11-13T08:28:32,798 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ec42fb5893064a8a979ba20bab66d0a1, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731486511922 2024-11-13T08:28:32,799 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8960a6bc4db048919d1fd9eb4617bccf, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T08:28:32,810 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): fb116209b3b52bde0867fb4d45873206#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T08:28:32,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741890_1068 (size=4626) 2024-11-13T08:28:32,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741890_1068 (size=4626) 2024-11-13T08:28:32,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741890_1068 (size=4626) 2024-11-13T08:28:32,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf1/7baa4d3a6eea43189e8518e02ca88915 as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/7baa4d3a6eea43189e8518e02ca88915 2024-11-13T08:28:32,839 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in fb116209b3b52bde0867fb4d45873206/cf1 of fb116209b3b52bde0867fb4d45873206 into 7baa4d3a6eea43189e8518e02ca88915(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T08:28:32,839 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for fb116209b3b52bde0867fb4d45873206: 2024-11-13T08:28:32,839 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T08:28:32,839 DEBUG [Time-limited test {}] regionserver.HStore(1541): fb116209b3b52bde0867fb4d45873206/cf2 is initiating major compaction (all files) 2024-11-13T08:28:32,839 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T08:28:32,839 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T08:28:32,839 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of fb116209b3b52bde0867fb4d45873206/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,840 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b] into tmpdir=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp, totalSize=4.8 K 2024-11-13T08:28:32,840 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e673242623ca4582ac9009c28defe46b, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T08:28:32,847 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): fb116209b3b52bde0867fb4d45873206#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T08:28:32,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741891_1069 (size=4592) 2024-11-13T08:28:32,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741891_1069 (size=4592) 2024-11-13T08:28:32,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741891_1069 (size=4592) 2024-11-13T08:28:32,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/.tmp/cf2/cfcaec2144c34d70afb9d70cc9df21f2 as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/cfcaec2144c34d70afb9d70cc9df21f2 2024-11-13T08:28:32,867 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in fb116209b3b52bde0867fb4d45873206/cf2 of fb116209b3b52bde0867fb4d45873206 into cfcaec2144c34d70afb9d70cc9df21f2(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T08:28:32,867 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for fb116209b3b52bde0867fb4d45873206: 2024-11-13T08:28:32,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=fb116209b3b52bde0867fb4d45873206, source=dd3a0865bb23,35727,1731486497292, destination=dd3a0865bb23,44073,1731486497392, warming up region on dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:32,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=fb116209b3b52bde0867fb4d45873206, source=dd3a0865bb23,35727,1731486497292, destination=dd3a0865bb23,44073,1731486497392, running balancer 2024-11-13T08:28:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE 2024-11-13T08:28:32,873 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE 2024-11-13T08:28:32,874 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=CLOSING, regionLocation=dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:32,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(7855): Warmup {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:32,875 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,877 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf1 2024-11-13T08:28:32,877 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:32,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE because future has completed 2024-11-13T08:28:32,878 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T08:28:32,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292}] 2024-11-13T08:28:32,885 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/7baa4d3a6eea43189e8518e02ca88915 2024-11-13T08:28:32,889 INFO [StoreFileOpener-fb116209b3b52bde0867fb4d45873206-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:32,889 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:32,893 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1 2024-11-13T08:28:32,894 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:32,894 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:32,895 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf2 2024-11-13T08:28:32,895 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:32,900 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/cfcaec2144c34d70afb9d70cc9df21f2 2024-11-13T08:28:32,904 INFO [StoreFileOpener-fb116209b3b52bde0867fb4d45873206-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:32,904 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:32,904 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:32,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44073 {}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486512905Disabling compacts and flushes for region at 1731486512905Disabling writes for close at 1731486512905Writing region close event to WAL at 1731486512906 (+1 ms)Closed at 1731486512906 2024-11-13T08:28:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T08:28:33,033 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,033 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T08:28:33,033 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:33,033 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,033 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,033 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:33,033 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,034 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1, hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf] to archive 2024-11-13T08:28:33,037 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T08:28:33,040 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1 to hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/ec42fb5893064a8a979ba20bab66d0a1 2024-11-13T08:28:33,042 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf to hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/8960a6bc4db048919d1fd9eb4617bccf 2024-11-13T08:28:33,055 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b] to archive 2024-11-13T08:28:33,056 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T08:28:33,057 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b to hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/e673242623ca4582ac9009c28defe46b 2024-11-13T08:28:33,063 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T08:28:33,063 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,064 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486513033Running coprocessor pre-close hooks at 1731486513033Disabling compacts and flushes for region at 1731486513033Disabling writes for close at 1731486513033Writing region close event to WAL at 1731486513059 (+26 ms)Running coprocessor post-close hooks at 1731486513063 (+4 ms)Closed at 1731486513063 2024-11-13T08:28:33,064 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding fb116209b3b52bde0867fb4d45873206 move to dd3a0865bb23,44073,1731486497392 record at close sequenceid=12 2024-11-13T08:28:33,066 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,067 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=CLOSED 2024-11-13T08:28:33,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292 because future has completed 2024-11-13T08:28:33,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T08:28:33,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292 in 193 msec 2024-11-13T08:28:33,074 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE; state=CLOSED, location=dd3a0865bb23,44073,1731486497392; forceNewPlan=false, retain=false 2024-11-13T08:28:33,225 INFO [dd3a0865bb23:37177 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T08:28:33,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPENING, regionLocation=dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:33,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE because future has completed 2024-11-13T08:28:33,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392}] 2024-11-13T08:28:33,386 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,387 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:33,387 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,387 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:33,388 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,388 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,389 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,391 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf1 2024-11-13T08:28:33,391 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:33,398 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/7baa4d3a6eea43189e8518e02ca88915 2024-11-13T08:28:33,398 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:33,398 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,399 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf2 2024-11-13T08:28:33,399 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:33,410 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/cfcaec2144c34d70afb9d70cc9df21f2 2024-11-13T08:28:33,410 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:33,411 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,412 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,413 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,414 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,414 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,415 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T08:28:33,417 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,418 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened fb116209b3b52bde0867fb4d45873206; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70949879, jitterRate=0.057235583662986755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T08:28:33,419 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,419 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for fb116209b3b52bde0867fb4d45873206: Running coprocessor pre-open hook at 1731486513388Writing region info on filesystem at 1731486513388Initializing all the Stores at 1731486513389 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486513389Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486513389Cleaning up temporary data from old regions at 1731486513415 (+26 ms)Running coprocessor post-open hooks at 1731486513419 (+4 ms)Region opened successfully at 1731486513419 2024-11-13T08:28:33,420 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., pid=12, masterSystemTime=1731486513382 2024-11-13T08:28:33,423 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,423 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPEN, openSeqNum=18, regionLocation=dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:33,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392 because future has completed 2024-11-13T08:28:33,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T08:28:33,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,44073,1731486497392 in 200 msec 2024-11-13T08:28:33,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, REOPEN/MOVE in 562 msec 2024-11-13T08:28:33,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T08:28:33,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T08:28:33,479 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server dd3a0865bb23,44073,1731486497392: testing ***** 2024-11-13T08:28:33,479 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T08:28:33,481 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T08:28:33,482 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T08:28:33,485 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T08:28:33,485 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T08:28:33,491 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 217019136 }, "NonHeapMemoryUsage": { "committed": 170590208, "init": 7667712, "max": -1, "used": 167920904 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "dd3a0865bb23", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "dd3a0865bb23", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62074", "tag.serverName": "dd3a0865bb23,44073,1731486497392", "tag.clusterId": "976bee63-cb40-472b-8f8e-0f050f8e2d48", "tag.Context": "regionserver", "tag.Hostname": "dd3a0865bb23", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731486497392, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T08:28:33,495 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37177 {}] master.MasterRpcServices(700): dd3a0865bb23,44073,1731486497392 reported a fatal error: ***** ABORTING region server dd3a0865bb23,44073,1731486497392: testing ***** 2024-11-13T08:28:33,497 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd3a0865bb23,44073,1731486497392' ***** 2024-11-13T08:28:33,497 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T08:28:33,497 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T08:28:33,498 INFO [RS:1;dd3a0865bb23:44073 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T08:28:33,498 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T08:28:33,498 INFO [RS:1;dd3a0865bb23:44073 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T08:28:33,498 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(3091): Received CLOSE for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,498 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(956): aborting server dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:33,498 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T08:28:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35727 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:34932 deadline: 1731486573498, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=44073 startCode=1731486497392. As of locationSeqNum=12. 2024-11-13T08:28:33,499 INFO [RS:1;dd3a0865bb23:44073 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dd3a0865bb23:44073. 2024-11-13T08:28:33,499 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:33,499 DEBUG [RS:1;dd3a0865bb23:44073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T08:28:33,499 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,499 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,499 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:33,499 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,499 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,35727,1731486497292, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,35727,1731486497292, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=44073 startCode=1731486497392. As of locationSeqNum=12. 2024-11-13T08:28:33,499 DEBUG [RS:1;dd3a0865bb23:44073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:33,499 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,35727,1731486497292, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=44073 startCode=1731486497392. As of locationSeqNum=12. 2024-11-13T08:28:33,499 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,35727,1731486497292, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dd3a0865bb23 port=44073 startCode=1731486497392. As of locationSeqNum=12. 2024-11-13T08:28:33,500 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T08:28:33,500 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1325): Online Regions={fb116209b3b52bde0867fb4d45873206=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.} 2024-11-13T08:28:33,500 DEBUG [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1351): Waiting on fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:33,501 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,501 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486513498Running coprocessor pre-close hooks at 1731486513499 (+1 ms)Disabling compacts and flushes for region at 1731486513499Disabling writes for close at 1731486513499Writing region close event to WAL at 1731486513500 (+1 ms)Running coprocessor post-close hooks at 1731486513500Closed at 1731486513501 (+1 ms) 2024-11-13T08:28:33,501 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:33,518 INFO [regionserver/dd3a0865bb23:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T08:28:33,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server dd3a0865bb23,44073,1731486497392 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:33,615 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server dd3a0865bb23,44073,1731486497392 aborting 2024-11-13T08:28:33,615 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server dd3a0865bb23,44073,1731486497392 aborting 2024-11-13T08:28:33,615 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=12 from cache 2024-11-13T08:28:33,663 INFO [regionserver/dd3a0865bb23:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T08:28:33,663 INFO [regionserver/dd3a0865bb23:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T08:28:33,701 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(976): stopping server dd3a0865bb23,44073,1731486497392; all regions closed. 2024-11-13T08:28:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741834_1010 (size=2142) 2024-11-13T08:28:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741834_1010 (size=2142) 2024-11-13T08:28:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741834_1010 (size=2142) 2024-11-13T08:28:33,705 DEBUG [RS:1;dd3a0865bb23:44073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:33,705 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T08:28:33,705 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T08:28:33,705 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.ChoreService(370): Chore service for: regionserver/dd3a0865bb23:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T08:28:33,706 INFO [regionserver/dd3a0865bb23:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T08:28:33,706 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T08:28:33,706 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T08:28:33,706 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T08:28:33,706 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T08:28:33,706 INFO [RS:1;dd3a0865bb23:44073 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44073 2024-11-13T08:28:33,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T08:28:33,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd3a0865bb23,44073,1731486497392 2024-11-13T08:28:33,714 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T08:28:33,715 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd3a0865bb23,44073,1731486497392] 2024-11-13T08:28:33,717 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd3a0865bb23,44073,1731486497392 already deleted, retry=false 2024-11-13T08:28:33,717 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of dd3a0865bb23,44073,1731486497392 on dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:33,722 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure dd3a0865bb23,44073,1731486497392, splitWal=true, meta=false 2024-11-13T08:28:33,724 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for dd3a0865bb23,44073,1731486497392 (carryingMeta=false) dd3a0865bb23,44073,1731486497392/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7026547[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T08:28:33,724 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure dd3a0865bb23,44073,1731486497392, splitWal=true, meta=false 2024-11-13T08:28:33,726 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): dd3a0865bb23,44073,1731486497392 had 1 regions 2024-11-13T08:28:33,727 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure dd3a0865bb23,44073,1731486497392, splitWal=true, meta=false, isMeta: false 2024-11-13T08:28:33,729 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting 2024-11-13T08:28:33,730 INFO [PEWorker-5 {}] master.SplitWALManager(105): dd3a0865bb23,44073,1731486497392 WAL count=1, meta=false 2024-11-13T08:28:33,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure dd3a0865bb23%2C44073%2C1731486497392.1731486498760}] 2024-11-13T08:28:33,739 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:33,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure dd3a0865bb23%2C44073%2C1731486497392.1731486498760, worker=dd3a0865bb23,45659,1731486497432}] 2024-11-13T08:28:33,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:33,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44073-0x10110711ec40002, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:33,817 INFO [RS:1;dd3a0865bb23:44073 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T08:28:33,817 INFO [RS:1;dd3a0865bb23:44073 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd3a0865bb23,44073,1731486497392; zookeeper connection closed. 2024-11-13T08:28:33,818 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3425db1a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3425db1a 2024-11-13T08:28:33,827 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18] 2024-11-13T08:28:33,829 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server dd3a0865bb23:44073 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: dd3a0865bb23/172.17.0.2:44073 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:33,830 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18, error=java.net.ConnectException: Call to address=dd3a0865bb23:44073 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: dd3a0865bb23/172.17.0.2:44073 2024-11-13T08:28:33,830 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-13T08:28:33,830 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18 from cache 2024-11-13T08:28:33,830 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address dd3a0865bb23:44073 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: dd3a0865bb23/172.17.0.2:44073 2024-11-13T08:28:33,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T08:28:33,922 INFO [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760, size=2.1 K (2142bytes) 2024-11-13T08:28:33,922 INFO [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 2024-11-13T08:28:33,922 INFO [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 after 0ms 2024-11-13T08:28:33,925 INFO [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 took 4ms 2024-11-13T08:28:33,934 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for fb116209b3b52bde0867fb4d45873206: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T08:28:33,934 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 so closing down 2024-11-13T08:28:33,934 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:33,934 INFO [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:33,934 INFO [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-13T08:28:33,934 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760, journal: Splitting hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760, size=2.1 K (2142bytes) at 1731486513922Finishing writing output for hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 so closing down at 1731486513934 (+12 ms)3 split writer threads finished at 1731486513934Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731486513934 2024-11-13T08:28:33,935 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 2024-11-13T08:28:33,936 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dd3a0865bb23:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T08:28:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37177 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T08:28:33,943 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting/dd3a0865bb23%2C44073%2C1731486497392.1731486498760 to hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs 2024-11-13T08:28:33,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T08:28:33,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure dd3a0865bb23%2C44073%2C1731486497392.1731486498760, worker=dd3a0865bb23,45659,1731486497432 in 202 msec 2024-11-13T08:28:33,947 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:33,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T08:28:33,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure dd3a0865bb23%2C44073%2C1731486497392.1731486498760, worker=dd3a0865bb23,45659,1731486497432 in 215 msec 2024-11-13T08:28:33,952 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting dir is empty, no logs to split. 2024-11-13T08:28:33,952 INFO [PEWorker-5 {}] master.SplitWALManager(105): dd3a0865bb23,44073,1731486497392 WAL count=0, meta=false 2024-11-13T08:28:33,952 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if dd3a0865bb23,44073,1731486497392 WAL splitting is done? wals=0, meta=false 2024-11-13T08:28:33,954 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for dd3a0865bb23,44073,1731486497392 failed, ignore...File hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/WALs/dd3a0865bb23,44073,1731486497392-splitting does not exist. 2024-11-13T08:28:33,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN}] 2024-11-13T08:28:33,957 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN 2024-11-13T08:28:33,958 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T08:28:34,108 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(204): Hosts are {dd3a0865bb23=0} racks are {/default-rack=0} 2024-11-13T08:28:34,109 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T08:28:34,109 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T08:28:34,109 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T08:28:34,109 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T08:28:34,109 INFO [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T08:28:34,109 INFO [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T08:28:34,109 DEBUG [dd3a0865bb23:37177 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T08:28:34,110 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPENING, regionLocation=dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:34,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN because future has completed 2024-11-13T08:28:34,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292}] 2024-11-13T08:28:34,136 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18] 2024-11-13T08:28:34,137 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to dd3a0865bb23:44073 this server is in the failed servers list 2024-11-13T08:28:34,137 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=dd3a0865bb23:44073 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: dd3a0865bb23:44073 2024-11-13T08:28:34,137 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: dd3a0865bb23:44073 2024-11-13T08:28:34,137 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,44073,1731486497392, seqNum=18 from cache 2024-11-13T08:28:34,270 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:34,270 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => fb116209b3b52bde0867fb4d45873206, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:34,270 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,271 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:34,271 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,271 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,272 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,273 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf1 2024-11-13T08:28:34,273 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,280 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf1/7baa4d3a6eea43189e8518e02ca88915 2024-11-13T08:28:34,280 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,281 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,281 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb116209b3b52bde0867fb4d45873206 columnFamilyName cf2 2024-11-13T08:28:34,281 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,288 DEBUG [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/cf2/cfcaec2144c34d70afb9d70cc9df21f2 2024-11-13T08:28:34,288 INFO [StoreOpener-fb116209b3b52bde0867fb4d45873206-1 {}] regionserver.HStore(327): Store=fb116209b3b52bde0867fb4d45873206/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,288 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,289 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,290 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,291 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,291 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,291 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T08:28:34,292 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,293 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened fb116209b3b52bde0867fb4d45873206; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69021602, jitterRate=0.02850201725959778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T08:28:34,293 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:34,294 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for fb116209b3b52bde0867fb4d45873206: Running coprocessor pre-open hook at 1731486514271Writing region info on filesystem at 1731486514271Initializing all the Stores at 1731486514272 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514272Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514272Cleaning up temporary data from old regions at 1731486514291 (+19 ms)Running coprocessor post-open hooks at 1731486514293 (+2 ms)Region opened successfully at 1731486514294 (+1 ms) 2024-11-13T08:28:34,295 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., pid=17, masterSystemTime=1731486514266 2024-11-13T08:28:34,297 DEBUG [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:34,297 INFO [RS_OPEN_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:34,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=fb116209b3b52bde0867fb4d45873206, regionState=OPEN, openSeqNum=18, regionLocation=dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:34,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292 because future has completed 2024-11-13T08:28:34,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T08:28:34,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure fb116209b3b52bde0867fb4d45873206, server=dd3a0865bb23,35727,1731486497292 in 189 msec 2024-11-13T08:28:34,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T08:28:34,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fb116209b3b52bde0867fb4d45873206, ASSIGN in 348 msec 2024-11-13T08:28:34,307 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server dd3a0865bb23,44073,1731486497392 after splitting done 2024-11-13T08:28:34,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure dd3a0865bb23,44073,1731486497392, splitWal=true, meta=false in 589 msec 2024-11-13T08:28:34,646 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206., hostname=dd3a0865bb23,35727,1731486497292, seqNum=18] 2024-11-13T08:28:34,662 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 408), OpenFileDescriptor=1052 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 146) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1609 (was 1757) 2024-11-13T08:28:34,662 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-11-13T08:28:34,675 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1052, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=1609 2024-11-13T08:28:34,675 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-11-13T08:28:34,691 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:34,693 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:34,694 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:34,696 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-00853072, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-00853072, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:34,709 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-00853072/hregion-00853072.1731486514696, exclude list is [], retry=0 2024-11-13T08:28:34,712 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:34,712 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:34,712 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:34,714 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-00853072/hregion-00853072.1731486514696 2024-11-13T08:28:34,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:34,715 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b40f3c529bd776c90d6eb24b57e61735, NAME => 'testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741893_1071 (size=67) 2024-11-13T08:28:34,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741893_1071 (size=67) 2024-11-13T08:28:34,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741893_1071 (size=67) 2024-11-13T08:28:34,726 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:34,727 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,728 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName a 2024-11-13T08:28:34,728 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,729 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,729 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,731 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName b 2024-11-13T08:28:34,731 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,731 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,731 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,733 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName c 2024-11-13T08:28:34,733 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,733 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,733 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,734 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,734 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,735 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,735 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,736 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:34,737 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,739 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:34,739 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b40f3c529bd776c90d6eb24b57e61735; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64198769, jitterRate=-0.043363794684410095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:34,740 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b40f3c529bd776c90d6eb24b57e61735: Writing region info on filesystem at 1731486514726Initializing all the Stores at 1731486514726Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514727 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514727Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514727Cleaning up temporary data from old regions at 1731486514735 (+8 ms)Region opened successfully at 1731486514740 (+5 ms) 2024-11-13T08:28:34,740 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b40f3c529bd776c90d6eb24b57e61735, disabling compactions & flushes 2024-11-13T08:28:34,740 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,740 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,740 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. after waiting 0 ms 2024-11-13T08:28:34,740 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,741 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,741 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b40f3c529bd776c90d6eb24b57e61735: Waiting for close lock at 1731486514740Disabling compacts and flushes for region at 1731486514740Disabling writes for close at 1731486514740Writing region close event to WAL at 1731486514741 (+1 ms)Closed at 1731486514741 2024-11-13T08:28:34,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741892_1070 (size=93) 2024-11-13T08:28:34,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741892_1070 (size=93) 2024-11-13T08:28:34,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741892_1070 (size=93) 2024-11-13T08:28:34,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:34,746 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-00853072:(num 1731486514696) 2024-11-13T08:28:34,746 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:34,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:34,762 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749, exclude list is [], retry=0 2024-11-13T08:28:34,765 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:34,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:34,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:34,768 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 2024-11-13T08:28:34,768 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:34,768 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b40f3c529bd776c90d6eb24b57e61735, NAME => 'testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:34,768 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:34,769 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,769 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,770 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,771 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName a 2024-11-13T08:28:34,771 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,772 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,772 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,773 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName b 2024-11-13T08:28:34,773 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,773 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,773 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,774 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName c 2024-11-13T08:28:34,774 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:34,774 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:34,775 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,776 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,778 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,778 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,779 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:34,780 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:34,781 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b40f3c529bd776c90d6eb24b57e61735; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71221408, jitterRate=0.061281681060791016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:34,782 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b40f3c529bd776c90d6eb24b57e61735: Writing region info on filesystem at 1731486514769Initializing all the Stores at 1731486514770 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514770Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514770Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486514770Cleaning up temporary data from old regions at 1731486514778 (+8 ms)Region opened successfully at 1731486514782 (+4 ms) 2024-11-13T08:28:34,815 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b40f3c529bd776c90d6eb24b57e61735 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T08:28:34,833 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/a/3757d506e94c4a648f7bce2e4bfd3e23 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731486514783/Put/seqid=0 2024-11-13T08:28:34,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741895_1073 (size=5958) 2024-11-13T08:28:34,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741895_1073 (size=5958) 2024-11-13T08:28:34,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741895_1073 (size=5958) 2024-11-13T08:28:34,843 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/a/3757d506e94c4a648f7bce2e4bfd3e23 2024-11-13T08:28:34,864 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/b/6c0794dfc4fc4cb3a83a000ad950b168 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731486514793/Put/seqid=0 2024-11-13T08:28:34,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741896_1074 (size=5958) 2024-11-13T08:28:34,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741896_1074 (size=5958) 2024-11-13T08:28:34,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741896_1074 (size=5958) 2024-11-13T08:28:34,873 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/b/6c0794dfc4fc4cb3a83a000ad950b168 2024-11-13T08:28:34,903 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/c/32ad6b2995f5432bb700293c0025bee7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731486514803/Put/seqid=0 2024-11-13T08:28:34,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741897_1075 (size=5958) 2024-11-13T08:28:34,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741897_1075 (size=5958) 2024-11-13T08:28:34,912 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/c/32ad6b2995f5432bb700293c0025bee7 2024-11-13T08:28:34,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741897_1075 (size=5958) 2024-11-13T08:28:34,920 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/a/3757d506e94c4a648f7bce2e4bfd3e23 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/a/3757d506e94c4a648f7bce2e4bfd3e23 2024-11-13T08:28:34,928 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/a/3757d506e94c4a648f7bce2e4bfd3e23, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T08:28:34,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/b/6c0794dfc4fc4cb3a83a000ad950b168 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/b/6c0794dfc4fc4cb3a83a000ad950b168 2024-11-13T08:28:34,936 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/b/6c0794dfc4fc4cb3a83a000ad950b168, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T08:28:34,937 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/c/32ad6b2995f5432bb700293c0025bee7 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/c/32ad6b2995f5432bb700293c0025bee7 2024-11-13T08:28:34,943 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/c/32ad6b2995f5432bb700293c0025bee7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T08:28:34,945 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b40f3c529bd776c90d6eb24b57e61735 in 129ms, sequenceid=33, compaction requested=false 2024-11-13T08:28:34,945 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b40f3c529bd776c90d6eb24b57e61735: 2024-11-13T08:28:34,945 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b40f3c529bd776c90d6eb24b57e61735, disabling compactions & flushes 2024-11-13T08:28:34,945 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,945 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,945 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. after waiting 0 ms 2024-11-13T08:28:34,945 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,946 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. 2024-11-13T08:28:34,946 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b40f3c529bd776c90d6eb24b57e61735: Waiting for close lock at 1731486514945Disabling compacts and flushes for region at 1731486514945Disabling writes for close at 1731486514945Writing region close event to WAL at 1731486514946 (+1 ms)Closed at 1731486514946 2024-11-13T08:28:34,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741894_1072 (size=5937) 2024-11-13T08:28:34,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741894_1072 (size=5937) 2024-11-13T08:28:34,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741894_1072 (size=5937) 2024-11-13T08:28:34,957 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/b/6c0794dfc4fc4cb3a83a000ad950b168 to hdfs://localhost:38569/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/b/6c0794dfc4fc4cb3a83a000ad950b168 2024-11-13T08:28:34,975 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749, size=5.8 K (5937bytes) 2024-11-13T08:28:34,975 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 2024-11-13T08:28:34,975 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 after 0ms 2024-11-13T08:28:34,977 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 took 3ms 2024-11-13T08:28:34,979 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 so closing down 2024-11-13T08:28:34,979 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:34,981 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731486514749.temp 2024-11-13T08:28:34,982 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000003-wal.1731486514749.temp 2024-11-13T08:28:34,982 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:34,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741898_1076 (size=5223) 2024-11-13T08:28:34,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741898_1076 (size=5223) 2024-11-13T08:28:34,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741898_1076 (size=5223) 2024-11-13T08:28:34,992 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000003-wal.1731486514749.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:34,994 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000003-wal.1731486514749.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000032 2024-11-13T08:28:34,994 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-13T08:28:34,994 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749, journal: Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749, size=5.8 K (5937bytes) at 1731486514975Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 so closing down at 1731486514979 (+4 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000003-wal.1731486514749.temp at 1731486514982 (+3 ms)3 split writer threads finished at 1731486514982Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000003-wal.1731486514749.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731486514992 (+10 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000003-wal.1731486514749.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000032 at 1731486514994 (+2 ms)Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731486514994 2024-11-13T08:28:34,996 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514749 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486514749 2024-11-13T08:28:34,997 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000032 2024-11-13T08:28:34,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:34,999 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:35,012 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514999, exclude list is [], retry=0 2024-11-13T08:28:35,015 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:35,016 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:35,016 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:35,018 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514999 2024-11-13T08:28:35,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:35,018 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b40f3c529bd776c90d6eb24b57e61735, NAME => 'testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:35,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:35,019 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,019 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,020 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,021 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName a 2024-11-13T08:28:35,021 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,027 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/a/3757d506e94c4a648f7bce2e4bfd3e23 2024-11-13T08:28:35,027 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,027 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,029 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName b 2024-11-13T08:28:35,029 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,029 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,030 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,030 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b40f3c529bd776c90d6eb24b57e61735 columnFamilyName c 2024-11-13T08:28:35,030 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,036 DEBUG [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/c/32ad6b2995f5432bb700293c0025bee7 2024-11-13T08:28:35,036 INFO [StoreOpener-b40f3c529bd776c90d6eb24b57e61735-1 {}] regionserver.HStore(327): Store=b40f3c529bd776c90d6eb24b57e61735/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,036 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,039 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000032 2024-11-13T08:28:35,043 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000032 2024-11-13T08:28:35,043 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b40f3c529bd776c90d6eb24b57e61735 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T08:28:35,058 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/b/71386c93e7dd42fd9c0530e644436086 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731486514793/Put/seqid=0 2024-11-13T08:28:35,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741900_1078 (size=5958) 2024-11-13T08:28:35,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741900_1078 (size=5958) 2024-11-13T08:28:35,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741900_1078 (size=5958) 2024-11-13T08:28:35,065 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/b/71386c93e7dd42fd9c0530e644436086 2024-11-13T08:28:35,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/.tmp/b/71386c93e7dd42fd9c0530e644436086 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/b/71386c93e7dd42fd9c0530e644436086 2024-11-13T08:28:35,075 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/b/71386c93e7dd42fd9c0530e644436086, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T08:28:35,075 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b40f3c529bd776c90d6eb24b57e61735 in 32ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T08:28:35,076 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/0000000000000000032 2024-11-13T08:28:35,077 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,077 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,078 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:35,079 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b40f3c529bd776c90d6eb24b57e61735 2024-11-13T08:28:35,081 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/b40f3c529bd776c90d6eb24b57e61735/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T08:28:35,082 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b40f3c529bd776c90d6eb24b57e61735; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59302171, jitterRate=-0.11632879078388214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:35,082 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b40f3c529bd776c90d6eb24b57e61735: Writing region info on filesystem at 1731486515019Initializing all the Stores at 1731486515020 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515020Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515020Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515020Obtaining lock to block concurrent updates at 1731486515043 (+23 ms)Preparing flush snapshotting stores in b40f3c529bd776c90d6eb24b57e61735 at 1731486515043Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731486515043Flushing stores of testReplayEditsWrittenViaHRegion,,1731486514691.b40f3c529bd776c90d6eb24b57e61735. at 1731486515043Flushing b40f3c529bd776c90d6eb24b57e61735/b: creating writer at 1731486515043Flushing b40f3c529bd776c90d6eb24b57e61735/b: appending metadata at 1731486515057 (+14 ms)Flushing b40f3c529bd776c90d6eb24b57e61735/b: closing flushed file at 1731486515057Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62175dc7: reopening flushed file at 1731486515070 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b40f3c529bd776c90d6eb24b57e61735 in 32ms, sequenceid=32, compaction requested=false; wal=null at 1731486515075 (+5 ms)Cleaning up temporary data from old regions at 1731486515077 (+2 ms)Region opened successfully at 1731486515082 (+5 ms) 2024-11-13T08:28:35,107 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:55992 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:56180 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:56614 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:56770 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:43222 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:43368 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1128 (was 1052) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=1604 (was 1609) 2024-11-13T08:28:35,108 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1128 is superior to 1024 2024-11-13T08:28:35,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1128, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=1603 2024-11-13T08:28:35,120 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1128 is superior to 1024 2024-11-13T08:28:35,135 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:35,137 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:35,138 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:35,140 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23092761, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-23092761, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:35,154 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23092761/hregion-23092761.1731486515140, exclude list is [], retry=0 2024-11-13T08:28:35,156 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:35,157 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:35,157 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:35,159 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23092761/hregion-23092761.1731486515140 2024-11-13T08:28:35,160 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:35,160 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e3a07689b94d552c6e23a8be22be0e17, NAME => 'testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:35,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741902_1080 (size=68) 2024-11-13T08:28:35,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741902_1080 (size=68) 2024-11-13T08:28:35,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741902_1080 (size=68) 2024-11-13T08:28:35,172 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:35,173 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,174 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName a 2024-11-13T08:28:35,174 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,175 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,175 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,176 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName b 2024-11-13T08:28:35,176 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,176 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,177 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,178 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName c 2024-11-13T08:28:35,178 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,178 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,178 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,179 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,179 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,180 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,180 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,181 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:35,182 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,184 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:35,184 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3a07689b94d552c6e23a8be22be0e17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60856926, jitterRate=-0.09316113591194153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3a07689b94d552c6e23a8be22be0e17: Writing region info on filesystem at 1731486515172Initializing all the Stores at 1731486515172Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515172Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515173 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515173Cleaning up temporary data from old regions at 1731486515180 (+7 ms)Region opened successfully at 1731486515185 (+5 ms) 2024-11-13T08:28:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e3a07689b94d552c6e23a8be22be0e17, disabling compactions & flushes 2024-11-13T08:28:35,185 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. after waiting 0 ms 2024-11-13T08:28:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:35,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:35,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e3a07689b94d552c6e23a8be22be0e17: Waiting for close lock at 1731486515185Disabling compacts and flushes for region at 1731486515185Disabling writes for close at 1731486515185Writing region close event to WAL at 1731486515185Closed at 1731486515186 (+1 ms) 2024-11-13T08:28:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741901_1079 (size=93) 2024-11-13T08:28:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741901_1079 (size=93) 2024-11-13T08:28:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741901_1079 (size=93) 2024-11-13T08:28:35,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:35,191 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23092761:(num 1731486515140) 2024-11-13T08:28:35,191 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:35,192 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:35,206 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193, exclude list is [], retry=0 2024-11-13T08:28:35,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:35,209 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:35,209 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:35,211 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 2024-11-13T08:28:35,211 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:35,281 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e3a07689b94d552c6e23a8be22be0e17, NAME => 'testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:35,284 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,284 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:35,284 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,284 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,286 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,287 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName a 2024-11-13T08:28:35,287 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,288 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,288 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,289 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName b 2024-11-13T08:28:35,289 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,289 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,289 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,290 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName c 2024-11-13T08:28:35,290 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:35,290 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:35,291 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,291 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,292 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,293 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,293 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,294 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:35,295 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,296 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3a07689b94d552c6e23a8be22be0e17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74776578, jitterRate=0.11425784230232239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:35,297 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:35,297 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3a07689b94d552c6e23a8be22be0e17: Running coprocessor pre-open hook at 1731486515284Writing region info on filesystem at 1731486515284Initializing all the Stores at 1731486515285 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515285Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515286 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486515286Cleaning up temporary data from old regions at 1731486515293 (+7 ms)Running coprocessor post-open hooks at 1731486515297 (+4 ms)Region opened successfully at 1731486515297 2024-11-13T08:28:35,308 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e3a07689b94d552c6e23a8be22be0e17 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T08:28:35,309 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:36,310 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:37,164 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T08:28:37,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T08:28:37,194 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:37,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T08:28:37,194 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:37,195 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T08:28:37,195 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:37,311 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:38,311 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:39,312 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:40,313 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:41,313 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:42,314 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:43,314 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:44,315 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e3a07689b94d552c6e23a8be22be0e17/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:44,316 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e3a07689b94d552c6e23a8be22be0e17: 2024-11-13T08:28:44,316 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:44,329 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e3a07689b94d552c6e23a8be22be0e17: 2024-11-13T08:28:44,330 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T08:28:44,330 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e3a07689b94d552c6e23a8be22be0e17, disabling compactions & flushes 2024-11-13T08:28:44,330 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:44,330 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:44,330 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. after waiting 0 ms 2024-11-13T08:28:44,330 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:44,330 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:44,331 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. 2024-11-13T08:28:44,331 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e3a07689b94d552c6e23a8be22be0e17: Waiting for close lock at 1731486524330Running coprocessor pre-close hooks at 1731486524330Disabling compacts and flushes for region at 1731486524330Disabling writes for close at 1731486524330Writing region close event to WAL at 1731486524330Running coprocessor post-close hooks at 1731486524330Closed at 1731486524331 (+1 ms) 2024-11-13T08:28:44,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741903_1081 (size=3941) 2024-11-13T08:28:44,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741903_1081 (size=3941) 2024-11-13T08:28:44,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741903_1081 (size=3941) 2024-11-13T08:28:44,350 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193, size=3.8 K (3941bytes) 2024-11-13T08:28:44,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 2024-11-13T08:28:44,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 after 0ms 2024-11-13T08:28:44,354 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 took 4ms 2024-11-13T08:28:44,356 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 so closing down 2024-11-13T08:28:44,356 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:44,357 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731486515193.temp 2024-11-13T08:28:44,358 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000004-wal.1731486515193.temp 2024-11-13T08:28:44,358 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:44,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741904_1082 (size=2983) 2024-11-13T08:28:44,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741904_1082 (size=2983) 2024-11-13T08:28:44,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741904_1082 (size=2983) 2024-11-13T08:28:44,367 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000004-wal.1731486515193.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:44,368 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000004-wal.1731486515193.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000026 2024-11-13T08:28:44,369 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-13T08:28:44,369 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193, journal: Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193, size=3.8 K (3941bytes) at 1731486524350Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 so closing down at 1731486524356 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000004-wal.1731486515193.temp at 1731486524358 (+2 ms)3 split writer threads finished at 1731486524358Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000004-wal.1731486515193.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731486524367 (+9 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000004-wal.1731486515193.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000026 at 1731486524368 (+1 ms)Processed 23 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731486524369 (+1 ms) 2024-11-13T08:28:44,370 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486515193 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486515193 2024-11-13T08:28:44,371 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000026 2024-11-13T08:28:44,371 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:44,373 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:44,386 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486524373, exclude list is [], retry=0 2024-11-13T08:28:44,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:44,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:44,390 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:44,391 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486524373 2024-11-13T08:28:44,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:44,392 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e3a07689b94d552c6e23a8be22be0e17, NAME => 'testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:44,392 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,393 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:44,393 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,393 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,395 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,396 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName a 2024-11-13T08:28:44,396 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:44,396 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:44,396 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,397 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName b 2024-11-13T08:28:44,397 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:44,398 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:44,398 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,399 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a07689b94d552c6e23a8be22be0e17 columnFamilyName c 2024-11-13T08:28:44,399 DEBUG [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:44,399 INFO [StoreOpener-e3a07689b94d552c6e23a8be22be0e17-1 {}] regionserver.HStore(327): Store=e3a07689b94d552c6e23a8be22be0e17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:44,399 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,400 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,401 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,402 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000026 2024-11-13T08:28:44,404 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000026 2024-11-13T08:28:44,405 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e3a07689b94d552c6e23a8be22be0e17 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T08:28:44,419 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/a/4a181b2f98aa493185a8afe161877902 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731486524321/Put/seqid=0 2024-11-13T08:28:44,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741906_1084 (size=5523) 2024-11-13T08:28:44,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741906_1084 (size=5523) 2024-11-13T08:28:44,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741906_1084 (size=5523) 2024-11-13T08:28:44,427 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/a/4a181b2f98aa493185a8afe161877902 2024-11-13T08:28:44,447 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/b/fa87070fc04742e8ab2a2d2d2aeba52e is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731486524317/Put/seqid=0 2024-11-13T08:28:44,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741907_1085 (size=5524) 2024-11-13T08:28:44,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741907_1085 (size=5524) 2024-11-13T08:28:44,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741907_1085 (size=5524) 2024-11-13T08:28:44,455 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/b/fa87070fc04742e8ab2a2d2d2aeba52e 2024-11-13T08:28:44,478 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/c/793b905d6ccc47dfa6bd615cdee0a146 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731486524319/Put/seqid=0 2024-11-13T08:28:44,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741908_1086 (size=5457) 2024-11-13T08:28:44,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741908_1086 (size=5457) 2024-11-13T08:28:44,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741908_1086 (size=5457) 2024-11-13T08:28:44,485 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/c/793b905d6ccc47dfa6bd615cdee0a146 2024-11-13T08:28:44,491 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/a/4a181b2f98aa493185a8afe161877902 as hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/a/4a181b2f98aa493185a8afe161877902 2024-11-13T08:28:44,496 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/a/4a181b2f98aa493185a8afe161877902, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T08:28:44,497 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/b/fa87070fc04742e8ab2a2d2d2aeba52e as hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/b/fa87070fc04742e8ab2a2d2d2aeba52e 2024-11-13T08:28:44,501 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/b/fa87070fc04742e8ab2a2d2d2aeba52e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T08:28:44,502 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/.tmp/c/793b905d6ccc47dfa6bd615cdee0a146 as hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/c/793b905d6ccc47dfa6bd615cdee0a146 2024-11-13T08:28:44,507 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/c/793b905d6ccc47dfa6bd615cdee0a146, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T08:28:44,507 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e3a07689b94d552c6e23a8be22be0e17 in 102ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T08:28:44,508 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/0000000000000000026 2024-11-13T08:28:44,509 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,509 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,510 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:44,511 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,513 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsAfterAbortingFlush/e3a07689b94d552c6e23a8be22be0e17/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T08:28:44,514 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3a07689b94d552c6e23a8be22be0e17; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72990228, jitterRate=0.08763915300369263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:44,514 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e3a07689b94d552c6e23a8be22be0e17 2024-11-13T08:28:44,514 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3a07689b94d552c6e23a8be22be0e17: Running coprocessor pre-open hook at 1731486524393Writing region info on filesystem at 1731486524393Initializing all the Stores at 1731486524394 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486524394Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486524394Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486524394Obtaining lock to block concurrent updates at 1731486524405 (+11 ms)Preparing flush snapshotting stores in e3a07689b94d552c6e23a8be22be0e17 at 1731486524405Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731486524405Flushing stores of testReplayEditsAfterAbortingFlush,,1731486515136.e3a07689b94d552c6e23a8be22be0e17. at 1731486524405Flushing e3a07689b94d552c6e23a8be22be0e17/a: creating writer at 1731486524405Flushing e3a07689b94d552c6e23a8be22be0e17/a: appending metadata at 1731486524419 (+14 ms)Flushing e3a07689b94d552c6e23a8be22be0e17/a: closing flushed file at 1731486524419Flushing e3a07689b94d552c6e23a8be22be0e17/b: creating writer at 1731486524432 (+13 ms)Flushing e3a07689b94d552c6e23a8be22be0e17/b: appending metadata at 1731486524446 (+14 ms)Flushing e3a07689b94d552c6e23a8be22be0e17/b: closing flushed file at 1731486524446Flushing e3a07689b94d552c6e23a8be22be0e17/c: creating writer at 1731486524462 (+16 ms)Flushing e3a07689b94d552c6e23a8be22be0e17/c: appending metadata at 1731486524478 (+16 ms)Flushing e3a07689b94d552c6e23a8be22be0e17/c: closing flushed file at 1731486524478Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@109158b3: reopening flushed file at 1731486524490 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b4122c: reopening flushed file at 1731486524496 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25e0cd94: reopening flushed file at 1731486524502 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e3a07689b94d552c6e23a8be22be0e17 in 102ms, sequenceid=26, compaction requested=false; wal=null at 1731486524507 (+5 ms)Cleaning up temporary data from old regions at 1731486524509 (+2 ms)Running coprocessor post-open hooks at 1731486524514 (+5 ms)Region opened successfully at 1731486524514 2024-11-13T08:28:44,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 417), OpenFileDescriptor=1182 (was 1128) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=1577 (was 1603) 2024-11-13T08:28:44,536 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-13T08:28:44,548 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=1576 2024-11-13T08:28:44,549 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-13T08:28:44,564 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:44,565 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:44,566 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:44,568 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-52583940, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-52583940, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:44,581 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-52583940/hregion-52583940.1731486524569, exclude list is [], retry=0 2024-11-13T08:28:44,583 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:44,583 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:44,584 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:44,586 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-52583940/hregion-52583940.1731486524569 2024-11-13T08:28:44,586 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:44,586 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b4c161ce26613d6d772bc57f70ce6963, NAME => 'testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:44,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741910_1088 (size=61) 2024-11-13T08:28:44,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741910_1088 (size=61) 2024-11-13T08:28:44,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741910_1088 (size=61) 2024-11-13T08:28:44,595 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:44,596 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,597 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4c161ce26613d6d772bc57f70ce6963 columnFamilyName a 2024-11-13T08:28:44,597 DEBUG [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:44,598 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(327): Store=b4c161ce26613d6d772bc57f70ce6963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:44,598 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,599 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,599 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,600 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,602 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:44,603 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4c161ce26613d6d772bc57f70ce6963; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71645003, jitterRate=0.06759373843669891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:44,603 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4c161ce26613d6d772bc57f70ce6963: Writing region info on filesystem at 1731486524595Initializing all the Stores at 1731486524596 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486524596Cleaning up temporary data from old regions at 1731486524599 (+3 ms)Region opened successfully at 1731486524603 (+4 ms) 2024-11-13T08:28:44,603 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b4c161ce26613d6d772bc57f70ce6963, disabling compactions & flushes 2024-11-13T08:28:44,603 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,603 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,603 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. after waiting 0 ms 2024-11-13T08:28:44,603 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,604 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,604 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b4c161ce26613d6d772bc57f70ce6963: Waiting for close lock at 1731486524603Disabling compacts and flushes for region at 1731486524603Disabling writes for close at 1731486524603Writing region close event to WAL at 1731486524604 (+1 ms)Closed at 1731486524604 2024-11-13T08:28:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741909_1087 (size=93) 2024-11-13T08:28:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741909_1087 (size=93) 2024-11-13T08:28:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741909_1087 (size=93) 2024-11-13T08:28:44,608 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:44,608 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-52583940:(num 1731486524569) 2024-11-13T08:28:44,608 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:44,610 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:44,627 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611, exclude list is [], retry=0 2024-11-13T08:28:44,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:44,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:44,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:44,632 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 2024-11-13T08:28:44,632 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:44,632 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b4c161ce26613d6d772bc57f70ce6963, NAME => 'testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:44,632 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:44,632 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,633 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,634 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,635 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4c161ce26613d6d772bc57f70ce6963 columnFamilyName a 2024-11-13T08:28:44,635 DEBUG [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:44,635 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(327): Store=b4c161ce26613d6d772bc57f70ce6963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:44,635 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,636 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,637 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,637 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,637 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,639 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,640 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4c161ce26613d6d772bc57f70ce6963; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63054661, jitterRate=-0.06041233241558075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:44,640 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4c161ce26613d6d772bc57f70ce6963: Writing region info on filesystem at 1731486524633Initializing all the Stores at 1731486524633Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486524633Cleaning up temporary data from old regions at 1731486524637 (+4 ms)Region opened successfully at 1731486524640 (+3 ms) 2024-11-13T08:28:44,648 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b4c161ce26613d6d772bc57f70ce6963, disabling compactions & flushes 2024-11-13T08:28:44,648 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,648 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,648 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. after waiting 0 ms 2024-11-13T08:28:44,648 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,649 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,649 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. 2024-11-13T08:28:44,649 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b4c161ce26613d6d772bc57f70ce6963: Waiting for close lock at 1731486524648Disabling compacts and flushes for region at 1731486524648Disabling writes for close at 1731486524648Writing region close event to WAL at 1731486524649 (+1 ms)Closed at 1731486524649 2024-11-13T08:28:44,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741911_1089 (size=1623) 2024-11-13T08:28:44,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741911_1089 (size=1623) 2024-11-13T08:28:44,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741911_1089 (size=1623) 2024-11-13T08:28:44,668 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611, size=1.6 K (1623bytes) 2024-11-13T08:28:44,668 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 2024-11-13T08:28:44,668 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 after 0ms 2024-11-13T08:28:44,670 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 took 2ms 2024-11-13T08:28:44,672 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 so closing down 2024-11-13T08:28:44,672 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:44,673 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731486524611.temp 2024-11-13T08:28:44,674 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000003-wal.1731486524611.temp 2024-11-13T08:28:44,675 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:44,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741912_1090 (size=1623) 2024-11-13T08:28:44,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741912_1090 (size=1623) 2024-11-13T08:28:44,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741912_1090 (size=1623) 2024-11-13T08:28:44,682 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000003-wal.1731486524611.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:44,683 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000003-wal.1731486524611.temp to hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000012 2024-11-13T08:28:44,683 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-13T08:28:44,683 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611, journal: Splitting hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611, size=1.6 K (1623bytes) at 1731486524668Finishing writing output for hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 so closing down at 1731486524672 (+4 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000003-wal.1731486524611.temp at 1731486524674 (+2 ms)3 split writer threads finished at 1731486524675 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000003-wal.1731486524611.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731486524682 (+7 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000003-wal.1731486524611.temp to hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000012 at 1731486524683 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731486524683 2024-11-13T08:28:44,685 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524611 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486524611 2024-11-13T08:28:44,685 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000012 2024-11-13T08:28:44,974 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:44,976 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:44,989 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524976, exclude list is [], retry=0 2024-11-13T08:28:44,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:44,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:44,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:44,994 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524976 2024-11-13T08:28:44,994 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:44,995 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b4c161ce26613d6d772bc57f70ce6963, NAME => 'testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:44,995 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:44,995 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,995 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:44,998 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,000 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4c161ce26613d6d772bc57f70ce6963 columnFamilyName a 2024-11-13T08:28:45,000 DEBUG [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,001 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(327): Store=b4c161ce26613d6d772bc57f70ce6963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,001 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,003 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,004 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000012 2024-11-13T08:28:45,007 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000012 2024-11-13T08:28:45,007 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b4c161ce26613d6d772bc57f70ce6963 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T08:28:45,027 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/.tmp/a/0cb1485144e74fd9adb08bac7a3115e4 is 79, key is testDatalossWhenInputError/a:x0/1731486524640/Put/seqid=0 2024-11-13T08:28:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741914_1092 (size=5808) 2024-11-13T08:28:45,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741914_1092 (size=5808) 2024-11-13T08:28:45,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741914_1092 (size=5808) 2024-11-13T08:28:45,035 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/.tmp/a/0cb1485144e74fd9adb08bac7a3115e4 2024-11-13T08:28:45,045 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/.tmp/a/0cb1485144e74fd9adb08bac7a3115e4 as hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/a/0cb1485144e74fd9adb08bac7a3115e4 2024-11-13T08:28:45,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/a/0cb1485144e74fd9adb08bac7a3115e4, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T08:28:45,053 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b4c161ce26613d6d772bc57f70ce6963 in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T08:28:45,054 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/0000000000000000012 2024-11-13T08:28:45,055 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,055 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,057 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,059 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T08:28:45,060 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4c161ce26613d6d772bc57f70ce6963; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70477491, jitterRate=0.05019645392894745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:45,061 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4c161ce26613d6d772bc57f70ce6963: Writing region info on filesystem at 1731486524995Initializing all the Stores at 1731486524997 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486524997Obtaining lock to block concurrent updates at 1731486525007 (+10 ms)Preparing flush snapshotting stores in b4c161ce26613d6d772bc57f70ce6963 at 1731486525007Finished memstore snapshotting testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731486525007Flushing stores of testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963. at 1731486525007Flushing b4c161ce26613d6d772bc57f70ce6963/a: creating writer at 1731486525007Flushing b4c161ce26613d6d772bc57f70ce6963/a: appending metadata at 1731486525026 (+19 ms)Flushing b4c161ce26613d6d772bc57f70ce6963/a: closing flushed file at 1731486525026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33728f61: reopening flushed file at 1731486525043 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b4c161ce26613d6d772bc57f70ce6963 in 46ms, sequenceid=12, compaction requested=false; wal=null at 1731486525053 (+10 ms)Cleaning up temporary data from old regions at 1731486525055 (+2 ms)Region opened successfully at 1731486525061 (+6 ms) 2024-11-13T08:28:45,064 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b4c161ce26613d6d772bc57f70ce6963, NAME => 'testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:45,064 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731486524564.b4c161ce26613d6d772bc57f70ce6963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:45,064 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,064 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,065 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,066 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4c161ce26613d6d772bc57f70ce6963 columnFamilyName a 2024-11-13T08:28:45,066 DEBUG [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,071 DEBUG [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/a/0cb1485144e74fd9adb08bac7a3115e4 2024-11-13T08:28:45,071 INFO [StoreOpener-b4c161ce26613d6d772bc57f70ce6963-1 {}] regionserver.HStore(327): Store=b4c161ce26613d6d772bc57f70ce6963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,071 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,073 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,074 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,074 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,076 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4c161ce26613d6d772bc57f70ce6963 2024-11-13T08:28:45,078 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testDatalossWhenInputError/b4c161ce26613d6d772bc57f70ce6963/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T08:28:45,079 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4c161ce26613d6d772bc57f70ce6963; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67688145, jitterRate=0.008631959557533264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T08:28:45,079 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4c161ce26613d6d772bc57f70ce6963: Writing region info on filesystem at 1731486525064Initializing all the Stores at 1731486525065 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525065Cleaning up temporary data from old regions at 1731486525074 (+9 ms)Region opened successfully at 1731486525079 (+5 ms) 2024-11-13T08:28:45,099 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:46032 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:44496 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:44590 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:49312 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:49380 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-433341908-172.17.0.2-1731486493558:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=1572 (was 1576) 2024-11-13T08:28:45,099 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T08:28:45,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=1571 2024-11-13T08:28:45,112 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T08:28:45,130 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:45,132 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:45,133 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:45,136 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-48908586, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-48908586, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:45,157 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-48908586/hregion-48908586.1731486525137, exclude list is [], retry=0 2024-11-13T08:28:45,161 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:45,161 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:45,161 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:45,164 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-48908586/hregion-48908586.1731486525137 2024-11-13T08:28:45,164 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-13T08:28:45,164 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a8860da1def4e2e1f5258edfb43b5ba5, NAME => 'testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:45,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741916_1094 (size=63) 2024-11-13T08:28:45,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741916_1094 (size=63) 2024-11-13T08:28:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741916_1094 (size=63) 2024-11-13T08:28:45,172 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:45,174 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,175 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName a 2024-11-13T08:28:45,175 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,176 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,176 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,177 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName b 2024-11-13T08:28:45,177 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,178 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,178 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,179 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName c 2024-11-13T08:28:45,179 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,180 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,180 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,180 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,181 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,182 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,182 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,183 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:45,184 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,186 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:45,186 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8860da1def4e2e1f5258edfb43b5ba5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70404916, jitterRate=0.049115002155303955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:45,187 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8860da1def4e2e1f5258edfb43b5ba5: Writing region info on filesystem at 1731486525172Initializing all the Stores at 1731486525173 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525173Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525174 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525174Cleaning up temporary data from old regions at 1731486525182 (+8 ms)Region opened successfully at 1731486525187 (+5 ms) 2024-11-13T08:28:45,187 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a8860da1def4e2e1f5258edfb43b5ba5, disabling compactions & flushes 2024-11-13T08:28:45,187 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:45,187 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:45,187 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. after waiting 0 ms 2024-11-13T08:28:45,187 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:45,187 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:45,187 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a8860da1def4e2e1f5258edfb43b5ba5: Waiting for close lock at 1731486525187Disabling compacts and flushes for region at 1731486525187Disabling writes for close at 1731486525187Writing region close event to WAL at 1731486525187Closed at 1731486525187 2024-11-13T08:28:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741915_1093 (size=93) 2024-11-13T08:28:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741915_1093 (size=93) 2024-11-13T08:28:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741915_1093 (size=93) 2024-11-13T08:28:45,192 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:45,192 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-48908586:(num 1731486525137) 2024-11-13T08:28:45,192 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:45,194 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:45,207 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194, exclude list is [], retry=0 2024-11-13T08:28:45,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:45,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:45,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:45,212 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 2024-11-13T08:28:45,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:45,212 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a8860da1def4e2e1f5258edfb43b5ba5, NAME => 'testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:45,212 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:45,212 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,212 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,214 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,215 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName a 2024-11-13T08:28:45,215 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,215 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,215 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,216 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName b 2024-11-13T08:28:45,216 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,217 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,217 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,218 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName c 2024-11-13T08:28:45,218 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:45,218 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:45,218 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,221 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,221 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,221 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:45,223 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:45,224 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8860da1def4e2e1f5258edfb43b5ba5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66668076, jitterRate=-0.006568253040313721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:45,224 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8860da1def4e2e1f5258edfb43b5ba5: Writing region info on filesystem at 1731486525213Initializing all the Stores at 1731486525213Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525213Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525214 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486525214Cleaning up temporary data from old regions at 1731486525221 (+7 ms)Region opened successfully at 1731486525224 (+3 ms) 2024-11-13T08:28:45,228 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731486525228/Put/seqid=0 2024-11-13T08:28:45,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741918_1096 (size=4875) 2024-11-13T08:28:45,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741918_1096 (size=4875) 2024-11-13T08:28:45,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741918_1096 (size=4875) 2024-11-13T08:28:45,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731486525238/Put/seqid=0 2024-11-13T08:28:45,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741919_1097 (size=4875) 2024-11-13T08:28:45,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741919_1097 (size=4875) 2024-11-13T08:28:45,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741919_1097 (size=4875) 2024-11-13T08:28:45,250 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731486525250/Put/seqid=0 2024-11-13T08:28:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741920_1098 (size=4875) 2024-11-13T08:28:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741920_1098 (size=4875) 2024-11-13T08:28:45,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741920_1098 (size=4875) 2024-11-13T08:28:45,261 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in a8860da1def4e2e1f5258edfb43b5ba5/a 2024-11-13T08:28:45,265 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T08:28:45,265 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T08:28:45,265 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in a8860da1def4e2e1f5258edfb43b5ba5/a 2024-11-13T08:28:45,269 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T08:28:45,269 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T08:28:45,269 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in a8860da1def4e2e1f5258edfb43b5ba5/a 2024-11-13T08:28:45,272 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T08:28:45,272 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T08:28:45,273 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a8860da1def4e2e1f5258edfb43b5ba5 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T08:28:45,289 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/.tmp/a/a5e28ad466fb409da8da82881fab4432 is 55, key is testCompactedBulkLoadedFiles/a:a/1731486525225/Put/seqid=0 2024-11-13T08:28:45,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741921_1099 (size=5107) 2024-11-13T08:28:45,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741921_1099 (size=5107) 2024-11-13T08:28:45,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741921_1099 (size=5107) 2024-11-13T08:28:45,296 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/.tmp/a/a5e28ad466fb409da8da82881fab4432 2024-11-13T08:28:45,301 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/.tmp/a/a5e28ad466fb409da8da82881fab4432 as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432 2024-11-13T08:28:45,306 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T08:28:45,307 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for a8860da1def4e2e1f5258edfb43b5ba5 in 34ms, sequenceid=4, compaction requested=false 2024-11-13T08:28:45,307 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a8860da1def4e2e1f5258edfb43b5ba5: 2024-11-13T08:28:45,308 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ 2024-11-13T08:28:45,309 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ 2024-11-13T08:28:45,310 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ 2024-11-13T08:28:45,310 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile0 into a8860da1def4e2e1f5258edfb43b5ba5/a as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ - updating store file list. 2024-11-13T08:28:45,315 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e7911649ffbf435e9d135384f19e5560_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:45,315 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ into a8860da1def4e2e1f5258edfb43b5ba5/a 2024-11-13T08:28:45,315 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile0 into a8860da1def4e2e1f5258edfb43b5ba5/a (new location: hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_) 2024-11-13T08:28:45,316 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile1 into a8860da1def4e2e1f5258edfb43b5ba5/a as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ - updating store file list. 2024-11-13T08:28:45,320 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:45,320 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ into a8860da1def4e2e1f5258edfb43b5ba5/a 2024-11-13T08:28:45,320 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile1 into a8860da1def4e2e1f5258edfb43b5ba5/a (new location: hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_) 2024-11-13T08:28:45,321 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile2 into a8860da1def4e2e1f5258edfb43b5ba5/a as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ - updating store file list. 2024-11-13T08:28:45,324 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fb57104785394b5182fa7371a5ece43b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:45,324 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ into a8860da1def4e2e1f5258edfb43b5ba5/a 2024-11-13T08:28:45,324 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38569/hbase/testCompactedBulkLoadedFiles/hfile2 into a8860da1def4e2e1f5258edfb43b5ba5/a (new location: hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_) 2024-11-13T08:28:45,331 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T08:28:45,332 DEBUG [Time-limited test {}] regionserver.HStore(1541): a8860da1def4e2e1f5258edfb43b5ba5/a is initiating major compaction (all files) 2024-11-13T08:28:45,332 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a8860da1def4e2e1f5258edfb43b5ba5/a in testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:45,332 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_] into tmpdir=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/.tmp, totalSize=19.3 K 2024-11-13T08:28:45,332 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a5e28ad466fb409da8da82881fab4432, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731486525225 2024-11-13T08:28:45,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e7911649ffbf435e9d135384f19e5560_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T08:28:45,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T08:28:45,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fb57104785394b5182fa7371a5ece43b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T08:28:45,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/.tmp/a/2d0fed773bfb448fa521e8857b934107 is 55, key is testCompactedBulkLoadedFiles/a:a/1731486525225/Put/seqid=0 2024-11-13T08:28:45,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741922_1100 (size=6154) 2024-11-13T08:28:45,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741922_1100 (size=6154) 2024-11-13T08:28:45,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741922_1100 (size=6154) 2024-11-13T08:28:45,365 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/.tmp/a/2d0fed773bfb448fa521e8857b934107 as hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/2d0fed773bfb448fa521e8857b934107 2024-11-13T08:28:45,371 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in a8860da1def4e2e1f5258edfb43b5ba5/a of a8860da1def4e2e1f5258edfb43b5ba5 into 2d0fed773bfb448fa521e8857b934107(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T08:28:45,371 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a8860da1def4e2e1f5258edfb43b5ba5: 2024-11-13T08:28:45,371 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T08:28:45,371 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T08:28:45,404 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194, size=0 (0bytes) 2024-11-13T08:28:45,404 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 might be still open, length is 0 2024-11-13T08:28:45,404 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 2024-11-13T08:28:45,404 WARN [IPC Server handler 2 on default port 38569 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-13T08:28:45,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 after 1ms 2024-11-13T08:28:45,737 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T08:28:47,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T08:28:47,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:47,007 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T08:28:47,007 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T08:28:48,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:49418 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34057:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49418 dst: /127.0.0.1:34057 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34057 remote=/127.0.0.1:49418]. Total timeout mills is 60000, 56981 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:48,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:46062 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:44221:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46062 dst: /127.0.0.1:44221 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:48,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:44632 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44632 dst: /127.0.0.1:34115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:48,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741917_1101 (size=1659) 2024-11-13T08:28:48,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741917_1101 (size=1659) 2024-11-13T08:28:48,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741917_1101 (size=1659) 2024-11-13T08:28:49,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 after 4001ms 2024-11-13T08:28:49,408 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 took 4005ms 2024-11-13T08:28:49,410 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194; continuing. 2024-11-13T08:28:49,410 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 so closing down 2024-11-13T08:28:49,410 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:49,411 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731486525194.temp 2024-11-13T08:28:49,413 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000003-wal.1731486525194.temp 2024-11-13T08:28:49,413 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741923_1102 (size=698) 2024-11-13T08:28:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741923_1102 (size=698) 2024-11-13T08:28:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741923_1102 (size=698) 2024-11-13T08:28:49,420 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000003-wal.1731486525194.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:49,422 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000003-wal.1731486525194.temp to hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000008 2024-11-13T08:28:49,422 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T08:28:49,422 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194, journal: Splitting hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194, size=0 (0bytes) at 1731486525404Finishing writing output for hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 so closing down at 1731486529410 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000003-wal.1731486525194.temp at 1731486529413 (+3 ms)3 split writer threads finished at 1731486529413Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000003-wal.1731486525194.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731486529420 (+7 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000003-wal.1731486525194.temp to hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000008 at 1731486529422 (+2 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194, size=0, length=0, corrupted=false, cancelled=false at 1731486529422 2024-11-13T08:28:49,423 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486525194 2024-11-13T08:28:49,424 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000008 2024-11-13T08:28:49,424 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:49,426 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:49,440 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486529426, exclude list is [], retry=0 2024-11-13T08:28:49,442 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:49,443 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:49,443 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:49,444 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486529426 2024-11-13T08:28:49,445 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a8860da1def4e2e1f5258edfb43b5ba5, NAME => 'testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,446 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,447 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName a 2024-11-13T08:28:49,447 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,454 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/2d0fed773bfb448fa521e8857b934107 2024-11-13T08:28:49,458 DEBUG [StoreFileOpener-a8860da1def4e2e1f5258edfb43b5ba5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:49,458 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ 2024-11-13T08:28:49,462 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432 2024-11-13T08:28:49,465 DEBUG [StoreFileOpener-a8860da1def4e2e1f5258edfb43b5ba5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e7911649ffbf435e9d135384f19e5560_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:49,465 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ 2024-11-13T08:28:49,469 DEBUG [StoreFileOpener-a8860da1def4e2e1f5258edfb43b5ba5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fb57104785394b5182fa7371a5ece43b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T08:28:49,469 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ 2024-11-13T08:28:49,469 WARN [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16e4db9 2024-11-13T08:28:49,469 WARN [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16e4db9 2024-11-13T08:28:49,469 WARN [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16e4db9 2024-11-13T08:28:49,469 WARN [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16e4db9 2024-11-13T08:28:49,469 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_] to archive 2024-11-13T08:28:49,470 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T08:28:49,472 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ to hdfs://localhost:38569/hbase/archive/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_ 2024-11-13T08:28:49,473 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432 to hdfs://localhost:38569/hbase/archive/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/a5e28ad466fb409da8da82881fab4432 2024-11-13T08:28:49,474 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ to hdfs://localhost:38569/hbase/archive/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/e7911649ffbf435e9d135384f19e5560_SeqId_4_ 2024-11-13T08:28:49,475 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ to hdfs://localhost:38569/hbase/archive/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/a/fb57104785394b5182fa7371a5ece43b_SeqId_4_ 2024-11-13T08:28:49,475 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,475 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,476 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName b 2024-11-13T08:28:49,476 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,477 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,477 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,478 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8860da1def4e2e1f5258edfb43b5ba5 columnFamilyName c 2024-11-13T08:28:49,478 DEBUG [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,478 INFO [StoreOpener-a8860da1def4e2e1f5258edfb43b5ba5-1 {}] regionserver.HStore(327): Store=a8860da1def4e2e1f5258edfb43b5ba5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,481 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000008 2024-11-13T08:28:49,484 DEBUG [Time-limited test {}] regionserver.HRegion(5836): a8860da1def4e2e1f5258edfb43b5ba5 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "a8860da1def4e2e1f5258edfb43b5ba5" family_name: "a" compaction_input: "a5e28ad466fb409da8da82881fab4432" compaction_input: "e7911649ffbf435e9d135384f19e5560_SeqId_4_" compaction_input: "381b2b3093e841c594b99b3fdda2b2c4_SeqId_4_" compaction_input: "fb57104785394b5182fa7371a5ece43b_SeqId_4_" compaction_output: "2d0fed773bfb448fa521e8857b934107" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T08:28:49,484 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T08:28:49,484 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000008 2024-11-13T08:28:49,485 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/0000000000000000008 2024-11-13T08:28:49,486 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,486 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,487 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:49,488 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8860da1def4e2e1f5258edfb43b5ba5 2024-11-13T08:28:49,491 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testCompactedBulkLoadedFiles/a8860da1def4e2e1f5258edfb43b5ba5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T08:28:49,491 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8860da1def4e2e1f5258edfb43b5ba5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67021366, jitterRate=-0.0013038218021392822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:49,492 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8860da1def4e2e1f5258edfb43b5ba5: Writing region info on filesystem at 1731486529445Initializing all the Stores at 1731486529446 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529446Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529446Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529446Cleaning up temporary data from old regions at 1731486529486 (+40 ms)Region opened successfully at 1731486529492 (+6 ms) 2024-11-13T08:28:49,494 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a8860da1def4e2e1f5258edfb43b5ba5, disabling compactions & flushes 2024-11-13T08:28:49,494 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:49,494 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:49,494 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. after waiting 0 ms 2024-11-13T08:28:49,494 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:49,495 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731486525131.a8860da1def4e2e1f5258edfb43b5ba5. 2024-11-13T08:28:49,495 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a8860da1def4e2e1f5258edfb43b5ba5: Waiting for close lock at 1731486529494Disabling compacts and flushes for region at 1731486529494Disabling writes for close at 1731486529494Writing region close event to WAL at 1731486529495 (+1 ms)Closed at 1731486529495 2024-11-13T08:28:49,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741924_1103 (size=93) 2024-11-13T08:28:49,498 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486529426 not finished, retry = 0 2024-11-13T08:28:49,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741924_1103 (size=93) 2024-11-13T08:28:49,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741924_1103 (size=93) 2024-11-13T08:28:49,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:49,601 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731486529426) 2024-11-13T08:28:49,615 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:38569 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38569 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35013 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1101882383_22 at /127.0.0.1:44716 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1101882383_22 at /127.0.0.1:46092 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1101882383_22 at /127.0.0.1:49482 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:35013 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=1515 (was 1571) 2024-11-13T08:28:49,615 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T08:28:49,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=1514 2024-11-13T08:28:49,628 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T08:28:49,643 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:49,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T08:28:49,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T08:28:49,648 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39986189, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/hregion-39986189, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:49,661 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39986189/hregion-39986189.1731486529648, exclude list is [], retry=0 2024-11-13T08:28:49,664 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:49,664 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:49,664 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:49,666 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39986189/hregion-39986189.1731486529648 2024-11-13T08:28:49,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:49,667 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4be64b4124944455c5491ae3539c6ab4, NAME => 'testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38569/hbase 2024-11-13T08:28:49,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741926_1105 (size=67) 2024-11-13T08:28:49,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741926_1105 (size=67) 2024-11-13T08:28:49,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741926_1105 (size=67) 2024-11-13T08:28:49,675 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:49,676 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,678 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName a 2024-11-13T08:28:49,678 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,678 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,678 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,679 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName b 2024-11-13T08:28:49,680 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,680 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,680 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,681 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName c 2024-11-13T08:28:49,681 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,682 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,682 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,684 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,684 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,684 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:49,685 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,687 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T08:28:49,688 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4be64b4124944455c5491ae3539c6ab4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63868215, jitterRate=-0.04828943312168121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:49,688 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4be64b4124944455c5491ae3539c6ab4: Writing region info on filesystem at 1731486529675Initializing all the Stores at 1731486529676 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529676Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529676Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529676Cleaning up temporary data from old regions at 1731486529684 (+8 ms)Region opened successfully at 1731486529688 (+4 ms) 2024-11-13T08:28:49,688 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4be64b4124944455c5491ae3539c6ab4, disabling compactions & flushes 2024-11-13T08:28:49,688 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,688 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,688 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. after waiting 0 ms 2024-11-13T08:28:49,688 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,689 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,689 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4be64b4124944455c5491ae3539c6ab4: Waiting for close lock at 1731486529688Disabling compacts and flushes for region at 1731486529688Disabling writes for close at 1731486529688Writing region close event to WAL at 1731486529689 (+1 ms)Closed at 1731486529689 2024-11-13T08:28:49,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741925_1104 (size=93) 2024-11-13T08:28:49,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741925_1104 (size=93) 2024-11-13T08:28:49,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741925_1104 (size=93) 2024-11-13T08:28:49,693 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:49,693 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39986189:(num 1731486529648) 2024-11-13T08:28:49,693 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:49,695 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:49,708 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695, exclude list is [], retry=0 2024-11-13T08:28:49,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:49,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:49,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:49,712 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 2024-11-13T08:28:49,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:49,714 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4be64b4124944455c5491ae3539c6ab4, NAME => 'testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:49,714 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:49,714 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,714 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,715 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,716 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName a 2024-11-13T08:28:49,716 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,717 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,717 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,718 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName b 2024-11-13T08:28:49,718 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,718 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,718 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,719 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName c 2024-11-13T08:28:49,719 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,720 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,720 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,722 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,723 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,723 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,723 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:49,725 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,726 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4be64b4124944455c5491ae3539c6ab4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69549660, jitterRate=0.036370694637298584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:49,726 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4be64b4124944455c5491ae3539c6ab4: Writing region info on filesystem at 1731486529714Initializing all the Stores at 1731486529715 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529715Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529715Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529715Cleaning up temporary data from old regions at 1731486529723 (+8 ms)Region opened successfully at 1731486529726 (+3 ms) 2024-11-13T08:28:49,734 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4be64b4124944455c5491ae3539c6ab4 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T08:28:49,751 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/a/bf5ede2052184b6ca9ce221962f5cc5d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731486529726/Put/seqid=0 2024-11-13T08:28:49,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741928_1107 (size=5958) 2024-11-13T08:28:49,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741928_1107 (size=5958) 2024-11-13T08:28:49,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741928_1107 (size=5958) 2024-11-13T08:28:49,758 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/a/bf5ede2052184b6ca9ce221962f5cc5d 2024-11-13T08:28:49,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/a/bf5ede2052184b6ca9ce221962f5cc5d as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/a/bf5ede2052184b6ca9ce221962f5cc5d 2024-11-13T08:28:49,768 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/a/bf5ede2052184b6ca9ce221962f5cc5d, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T08:28:49,769 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4be64b4124944455c5491ae3539c6ab4 in 35ms, sequenceid=13, compaction requested=false 2024-11-13T08:28:49,769 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4be64b4124944455c5491ae3539c6ab4: 2024-11-13T08:28:49,787 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4be64b4124944455c5491ae3539c6ab4, disabling compactions & flushes 2024-11-13T08:28:49,787 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,787 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,787 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. after waiting 0 ms 2024-11-13T08:28:49,787 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,787 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,787 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:49,787 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4be64b4124944455c5491ae3539c6ab4: Waiting for close lock at 1731486529786Disabling compacts and flushes for region at 1731486529786Disabling writes for close at 1731486529787 (+1 ms)Writing region close event to WAL at 1731486529787Closed at 1731486529787 2024-11-13T08:28:49,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741927_1106 (size=5869) 2024-11-13T08:28:49,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741927_1106 (size=5869) 2024-11-13T08:28:49,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741927_1106 (size=5869) 2024-11-13T08:28:49,806 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695, size=5.7 K (5869bytes) 2024-11-13T08:28:49,806 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 2024-11-13T08:28:49,806 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 after 0ms 2024-11-13T08:28:49,809 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 took 2ms 2024-11-13T08:28:49,810 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 so closing down 2024-11-13T08:28:49,810 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:49,812 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731486529695.temp 2024-11-13T08:28:49,813 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000003-wal.1731486529695.temp 2024-11-13T08:28:49,813 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:49,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741929_1108 (size=5223) 2024-11-13T08:28:49,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741929_1108 (size=5223) 2024-11-13T08:28:49,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741929_1108 (size=5223) 2024-11-13T08:28:49,822 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000003-wal.1731486529695.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:49,824 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000003-wal.1731486529695.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000035 2024-11-13T08:28:49,824 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-13T08:28:49,824 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695, journal: Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695, size=5.7 K (5869bytes) at 1731486529806Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 so closing down at 1731486529810 (+4 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000003-wal.1731486529695.temp at 1731486529813 (+3 ms)3 split writer threads finished at 1731486529813Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000003-wal.1731486529695.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731486529822 (+9 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000003-wal.1731486529695.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000035 at 1731486529824 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731486529824 2024-11-13T08:28:49,826 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529695 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486529695 2024-11-13T08:28:49,826 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000035 2024-11-13T08:28:49,827 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:49,828 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:49,842 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829, exclude list is [], retry=0 2024-11-13T08:28:49,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:49,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:49,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:49,846 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 2024-11-13T08:28:49,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:39765:39765)] 2024-11-13T08:28:49,848 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4be64b4124944455c5491ae3539c6ab4, NAME => 'testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.', STARTKEY => '', ENDKEY => ''} 2024-11-13T08:28:49,848 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:49,848 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,848 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,850 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,851 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName a 2024-11-13T08:28:49,851 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,852 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T08:28:49,857 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/a/bf5ede2052184b6ca9ce221962f5cc5d 2024-11-13T08:28:49,857 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,857 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,858 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName b 2024-11-13T08:28:49,858 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,858 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,858 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,859 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName c 2024-11-13T08:28:49,859 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:49,859 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:49,860 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,860 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,862 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,862 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000035 2024-11-13T08:28:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000035 2024-11-13T08:28:49,865 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4be64b4124944455c5491ae3539c6ab4 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T08:28:49,880 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/b/ba094d053c824937a8d60c8831243b98 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731486529769/Put/seqid=0 2024-11-13T08:28:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741931_1110 (size=5958) 2024-11-13T08:28:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741931_1110 (size=5958) 2024-11-13T08:28:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741931_1110 (size=5958) 2024-11-13T08:28:49,887 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/b/ba094d053c824937a8d60c8831243b98 2024-11-13T08:28:49,914 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/c/44002cb37f6a4166b17366c46d438147 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731486529777/Put/seqid=0 2024-11-13T08:28:49,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741932_1111 (size=5958) 2024-11-13T08:28:49,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741932_1111 (size=5958) 2024-11-13T08:28:49,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741932_1111 (size=5958) 2024-11-13T08:28:49,921 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/c/44002cb37f6a4166b17366c46d438147 2024-11-13T08:28:49,926 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/b/ba094d053c824937a8d60c8831243b98 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/b/ba094d053c824937a8d60c8831243b98 2024-11-13T08:28:49,931 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/b/ba094d053c824937a8d60c8831243b98, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T08:28:49,932 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/c/44002cb37f6a4166b17366c46d438147 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/c/44002cb37f6a4166b17366c46d438147 2024-11-13T08:28:49,937 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/c/44002cb37f6a4166b17366c46d438147, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T08:28:49,937 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4be64b4124944455c5491ae3539c6ab4 in 72ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T08:28:49,938 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000035 2024-11-13T08:28:49,938 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,938 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,939 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:49,940 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:49,942 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T08:28:49,943 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4be64b4124944455c5491ae3539c6ab4; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72436112, jitterRate=0.07938218116760254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:49,943 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4be64b4124944455c5491ae3539c6ab4: Writing region info on filesystem at 1731486529848Initializing all the Stores at 1731486529849 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529849Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529849Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486529850 (+1 ms)Obtaining lock to block concurrent updates at 1731486529865 (+15 ms)Preparing flush snapshotting stores in 4be64b4124944455c5491ae3539c6ab4 at 1731486529865Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731486529865Flushing stores of testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. at 1731486529865Flushing 4be64b4124944455c5491ae3539c6ab4/b: creating writer at 1731486529865Flushing 4be64b4124944455c5491ae3539c6ab4/b: appending metadata at 1731486529879 (+14 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/b: closing flushed file at 1731486529879Flushing 4be64b4124944455c5491ae3539c6ab4/c: creating writer at 1731486529892 (+13 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/c: appending metadata at 1731486529913 (+21 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/c: closing flushed file at 1731486529913Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14ac3ddf: reopening flushed file at 1731486529926 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21833ed3: reopening flushed file at 1731486529931 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4be64b4124944455c5491ae3539c6ab4 in 72ms, sequenceid=35, compaction requested=false; wal=null at 1731486529937 (+6 ms)Cleaning up temporary data from old regions at 1731486529938 (+1 ms)Region opened successfully at 1731486529943 (+5 ms) 2024-11-13T08:28:50,009 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829, size=0 (0bytes) 2024-11-13T08:28:50,009 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 might be still open, length is 0 2024-11-13T08:28:50,009 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 2024-11-13T08:28:50,010 WARN [IPC Server handler 2 on default port 38569 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-13T08:28:50,010 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 after 1ms 2024-11-13T08:28:51,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:49538 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34057:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49538 dst: /127.0.0.1:34057 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34057 remote=/127.0.0.1:49538]. Total timeout mills is 60000, 58448 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:51,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:44768 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44768 dst: /127.0.0.1:34115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:51,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2133564029_22 at /127.0.0.1:46190 [Receiving block BP-433341908-172.17.0.2-1731486493558:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:44221:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46190 dst: /127.0.0.1:44221 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:51,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741930_1112 (size=5215) 2024-11-13T08:28:51,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741930_1112 (size=5215) 2024-11-13T08:28:51,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741930_1112 (size=5215) 2024-11-13T08:28:54,011 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 after 4002ms 2024-11-13T08:28:54,014 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 took 4005ms 2024-11-13T08:28:54,016 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829; continuing. 2024-11-13T08:28:54,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 so closing down 2024-11-13T08:28:54,016 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T08:28:54,017 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731486529829.temp 2024-11-13T08:28:54,018 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000037-wal.1731486529829.temp 2024-11-13T08:28:54,019 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T08:28:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741933_1113 (size=5223) 2024-11-13T08:28:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741933_1113 (size=5223) 2024-11-13T08:28:54,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741933_1113 (size=5223) 2024-11-13T08:28:54,029 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000037-wal.1731486529829.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T08:28:54,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000037-wal.1731486529829.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000066 2024-11-13T08:28:54,030 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T08:28:54,031 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829, journal: Splitting hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829, size=0 (0bytes) at 1731486530009Finishing writing output for hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 so closing down at 1731486534016 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000037-wal.1731486529829.temp at 1731486534018 (+2 ms)3 split writer threads finished at 1731486534019 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000037-wal.1731486529829.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731486534029 (+10 ms)Rename recovered edits hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000037-wal.1731486529829.temp to hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000066 at 1731486534030 (+1 ms)Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829, size=0, length=0, corrupted=false, cancelled=false at 1731486534030 2024-11-13T08:28:54,032 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 to hdfs://localhost:38569/hbase/oldWALs/wal.1731486529829 2024-11-13T08:28:54,033 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000066 2024-11-13T08:28:54,033 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T08:28:54,035 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38569/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642, archiveDir=hdfs://localhost:38569/hbase/oldWALs, maxLogs=32 2024-11-13T08:28:54,049 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486534035, exclude list is [], retry=0 2024-11-13T08:28:54,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44221,DS-78f58666-22d4-4130-8364-a840009c3df9,DISK] 2024-11-13T08:28:54,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34057,DS-90e3298c-fbb8-45d6-828a-2d5a9bb298e4,DISK] 2024-11-13T08:28:54,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34115,DS-1b8f8a47-8ede-4400-a165-80b0a456379b,DISK] 2024-11-13T08:28:54,053 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486534035 2024-11-13T08:28:54,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39765:39765),(127.0.0.1/127.0.0.1:41159:41159),(127.0.0.1/127.0.0.1:37895:37895)] 2024-11-13T08:28:54,054 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T08:28:54,056 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,056 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName a 2024-11-13T08:28:54,056 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:54,061 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/a/bf5ede2052184b6ca9ce221962f5cc5d 2024-11-13T08:28:54,061 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:54,061 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,062 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName b 2024-11-13T08:28:54,062 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:54,066 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/b/ba094d053c824937a8d60c8831243b98 2024-11-13T08:28:54,066 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:54,066 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,067 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4be64b4124944455c5491ae3539c6ab4 columnFamilyName c 2024-11-13T08:28:54,067 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T08:28:54,071 DEBUG [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/c/44002cb37f6a4166b17366c46d438147 2024-11-13T08:28:54,071 INFO [StoreOpener-4be64b4124944455c5491ae3539c6ab4-1 {}] regionserver.HStore(327): Store=4be64b4124944455c5491ae3539c6ab4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T08:28:54,072 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,073 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,074 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000066 2024-11-13T08:28:54,082 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000066 2024-11-13T08:28:54,082 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4be64b4124944455c5491ae3539c6ab4 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T08:28:54,096 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/a/3ce6088f79744754aaabfbf13aabcc59 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731486529950/Put/seqid=0 2024-11-13T08:28:54,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741935_1115 (size=5958) 2024-11-13T08:28:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741935_1115 (size=5958) 2024-11-13T08:28:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741935_1115 (size=5958) 2024-11-13T08:28:54,103 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/a/3ce6088f79744754aaabfbf13aabcc59 2024-11-13T08:28:54,123 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/b/7447bcd2e7c6446bb8f7ee917190070a is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731486529958/Put/seqid=0 2024-11-13T08:28:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741936_1116 (size=5958) 2024-11-13T08:28:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741936_1116 (size=5958) 2024-11-13T08:28:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741936_1116 (size=5958) 2024-11-13T08:28:54,130 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/b/7447bcd2e7c6446bb8f7ee917190070a 2024-11-13T08:28:54,150 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/c/6c74455b1d6c45bfb9dcfa163a9b4174 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731486529964/Put/seqid=0 2024-11-13T08:28:54,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741937_1117 (size=5958) 2024-11-13T08:28:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741937_1117 (size=5958) 2024-11-13T08:28:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741937_1117 (size=5958) 2024-11-13T08:28:54,158 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/c/6c74455b1d6c45bfb9dcfa163a9b4174 2024-11-13T08:28:54,163 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/a/3ce6088f79744754aaabfbf13aabcc59 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/a/3ce6088f79744754aaabfbf13aabcc59 2024-11-13T08:28:54,168 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/a/3ce6088f79744754aaabfbf13aabcc59, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T08:28:54,168 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/b/7447bcd2e7c6446bb8f7ee917190070a as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/b/7447bcd2e7c6446bb8f7ee917190070a 2024-11-13T08:28:54,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/b/7447bcd2e7c6446bb8f7ee917190070a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T08:28:54,173 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/.tmp/c/6c74455b1d6c45bfb9dcfa163a9b4174 as hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/c/6c74455b1d6c45bfb9dcfa163a9b4174 2024-11-13T08:28:54,178 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/c/6c74455b1d6c45bfb9dcfa163a9b4174, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T08:28:54,179 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4be64b4124944455c5491ae3539c6ab4 in 97ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T08:28:54,179 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/0000000000000000066 2024-11-13T08:28:54,181 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,181 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,181 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T08:28:54,183 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4be64b4124944455c5491ae3539c6ab4 2024-11-13T08:28:54,185 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/hbase/data/default/testReplayEditsWrittenViaHRegion/4be64b4124944455c5491ae3539c6ab4/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T08:28:54,186 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4be64b4124944455c5491ae3539c6ab4; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68049132, jitterRate=0.014011085033416748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T08:28:54,186 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4be64b4124944455c5491ae3539c6ab4: Writing region info on filesystem at 1731486534054Initializing all the Stores at 1731486534055 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486534055Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486534055Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731486534055Obtaining lock to block concurrent updates at 1731486534082 (+27 ms)Preparing flush snapshotting stores in 4be64b4124944455c5491ae3539c6ab4 at 1731486534082Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731486534082Flushing stores of testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. at 1731486534082Flushing 4be64b4124944455c5491ae3539c6ab4/a: creating writer at 1731486534082Flushing 4be64b4124944455c5491ae3539c6ab4/a: appending metadata at 1731486534096 (+14 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/a: closing flushed file at 1731486534096Flushing 4be64b4124944455c5491ae3539c6ab4/b: creating writer at 1731486534108 (+12 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/b: appending metadata at 1731486534122 (+14 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/b: closing flushed file at 1731486534122Flushing 4be64b4124944455c5491ae3539c6ab4/c: creating writer at 1731486534136 (+14 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/c: appending metadata at 1731486534150 (+14 ms)Flushing 4be64b4124944455c5491ae3539c6ab4/c: closing flushed file at 1731486534150Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eb9b330: reopening flushed file at 1731486534162 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f750c76: reopening flushed file at 1731486534168 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6691e1b2: reopening flushed file at 1731486534173 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4be64b4124944455c5491ae3539c6ab4 in 97ms, sequenceid=66, compaction requested=false; wal=null at 1731486534179 (+6 ms)Cleaning up temporary data from old regions at 1731486534181 (+2 ms)Region opened successfully at 1731486534186 (+5 ms) 2024-11-13T08:28:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4be64b4124944455c5491ae3539c6ab4, disabling compactions & flushes 2024-11-13T08:28:54,200 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. after waiting 0 ms 2024-11-13T08:28:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:54,202 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731486529643.4be64b4124944455c5491ae3539c6ab4. 2024-11-13T08:28:54,202 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4be64b4124944455c5491ae3539c6ab4: Waiting for close lock at 1731486534200Disabling compacts and flushes for region at 1731486534200Disabling writes for close at 1731486534200Writing region close event to WAL at 1731486534202 (+2 ms)Closed at 1731486534202 2024-11-13T08:28:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741934_1114 (size=93) 2024-11-13T08:28:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741934_1114 (size=93) 2024-11-13T08:28:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741934_1114 (size=93) 2024-11-13T08:28:54,206 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T08:28:54,206 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731486534035) 2024-11-13T08:28:54,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=447 (was 438) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1835499349_22 at /127.0.0.1:54702 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1835499349_22 at /127.0.0.1:36934 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:38569 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1835499349_22 at /127.0.0.1:46374 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38569 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1416 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=115 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=1520 (was 1514) - AvailableMemoryMB LEAK? - 2024-11-13T08:28:54,224 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1416 is superior to 1024 2024-11-13T08:28:54,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T08:28:54,225 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T08:28:54,225 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T08:28:54,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T08:28:54,225 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T08:28:54,226 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=208037756, stopped=false 2024-11-13T08:28:54,226 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd3a0865bb23,37177,1731486496596 2024-11-13T08:28:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T08:28:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:54,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:54,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T08:28:54,229 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T08:28:54,229 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T08:28:54,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:54,229 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd3a0865bb23,35727,1731486497292' ***** 2024-11-13T08:28:54,229 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T08:28:54,229 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd3a0865bb23,45659,1731486497432' ***** 2024-11-13T08:28:54,229 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T08:28:54,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(3091): Received CLOSE for fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(959): stopping server dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:54,230 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T08:28:54,230 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd3a0865bb23:35727. 2024-11-13T08:28:54,230 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb116209b3b52bde0867fb4d45873206, disabling compactions & flushes 2024-11-13T08:28:54,230 DEBUG [RS:0;dd3a0865bb23:35727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T08:28:54,230 DEBUG [RS:0;dd3a0865bb23:35727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,230 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:54,230 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T08:28:54,230 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:54,230 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1325): Online Regions={fb116209b3b52bde0867fb4d45873206=testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206.} 2024-11-13T08:28:54,230 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. after waiting 0 ms 2024-11-13T08:28:54,230 DEBUG [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1351): Waiting on fb116209b3b52bde0867fb4d45873206 2024-11-13T08:28:54,230 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(959): stopping server dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;dd3a0865bb23:45659. 2024-11-13T08:28:54,231 DEBUG [RS:2;dd3a0865bb23:45659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T08:28:54,231 DEBUG [RS:2;dd3a0865bb23:45659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T08:28:54,231 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T08:28:54,231 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T08:28:54,232 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T08:28:54,232 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T08:28:54,232 DEBUG [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T08:28:54,232 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T08:28:54,232 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T08:28:54,232 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T08:28:54,232 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T08:28:54,232 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T08:28:54,232 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-13T08:28:54,232 INFO [regionserver/dd3a0865bb23:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T08:28:54,233 INFO [regionserver/dd3a0865bb23:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T08:28:54,237 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fb116209b3b52bde0867fb4d45873206/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T08:28:54,237 INFO [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:54,238 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb116209b3b52bde0867fb4d45873206: Waiting for close lock at 1731486534230Running coprocessor pre-close hooks at 1731486534230Disabling compacts and flushes for region at 1731486534230Disabling writes for close at 1731486534230Writing region close event to WAL at 1731486534232 (+2 ms)Running coprocessor post-close hooks at 1731486534237 (+5 ms)Closed at 1731486534237 2024-11-13T08:28:54,238 DEBUG [RS_CLOSE_REGION-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206. 2024-11-13T08:28:54,254 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/info/6d406f85b78f47fea0419add9f086da7 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731486511244.fb116209b3b52bde0867fb4d45873206./info:regioninfo/1731486514298/Put/seqid=0 2024-11-13T08:28:54,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741938_1118 (size=8243) 2024-11-13T08:28:54,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741938_1118 (size=8243) 2024-11-13T08:28:54,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741938_1118 (size=8243) 2024-11-13T08:28:54,261 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/info/6d406f85b78f47fea0419add9f086da7 2024-11-13T08:28:54,284 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/ns/89f815491f1540f3abd0bbf1f2deadc6 is 43, key is default/ns:d/1731486499270/Put/seqid=0 2024-11-13T08:28:54,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741939_1119 (size=5153) 2024-11-13T08:28:54,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741939_1119 (size=5153) 2024-11-13T08:28:54,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741939_1119 (size=5153) 2024-11-13T08:28:54,291 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/ns/89f815491f1540f3abd0bbf1f2deadc6 2024-11-13T08:28:54,311 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/table/d87554a8c48a46f49c62a94e135a13f2 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731486511674/Put/seqid=0 2024-11-13T08:28:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741940_1120 (size=5431) 2024-11-13T08:28:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741940_1120 (size=5431) 2024-11-13T08:28:54,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741940_1120 (size=5431) 2024-11-13T08:28:54,318 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/table/d87554a8c48a46f49c62a94e135a13f2 2024-11-13T08:28:54,324 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/info/6d406f85b78f47fea0419add9f086da7 as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/info/6d406f85b78f47fea0419add9f086da7 2024-11-13T08:28:54,330 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/info/6d406f85b78f47fea0419add9f086da7, entries=18, sequenceid=21, filesize=8.0 K 2024-11-13T08:28:54,330 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/ns/89f815491f1540f3abd0bbf1f2deadc6 as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/ns/89f815491f1540f3abd0bbf1f2deadc6 2024-11-13T08:28:54,336 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/ns/89f815491f1540f3abd0bbf1f2deadc6, entries=2, sequenceid=21, filesize=5.0 K 2024-11-13T08:28:54,337 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/.tmp/table/d87554a8c48a46f49c62a94e135a13f2 as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/table/d87554a8c48a46f49c62a94e135a13f2 2024-11-13T08:28:54,342 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/table/d87554a8c48a46f49c62a94e135a13f2, entries=2, sequenceid=21, filesize=5.3 K 2024-11-13T08:28:54,343 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false 2024-11-13T08:28:54,348 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-13T08:28:54,348 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T08:28:54,348 INFO [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T08:28:54,349 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731486534232Running coprocessor pre-close hooks at 1731486534232Disabling compacts and flushes for region at 1731486534232Disabling writes for close at 1731486534232Obtaining lock to block concurrent updates at 1731486534232Preparing flush snapshotting stores in 1588230740 at 1731486534232Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731486534233 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731486534233Flushing 1588230740/info: creating writer at 1731486534234 (+1 ms)Flushing 1588230740/info: appending metadata at 1731486534253 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731486534253Flushing 1588230740/ns: creating writer at 1731486534267 (+14 ms)Flushing 1588230740/ns: appending metadata at 1731486534283 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731486534283Flushing 1588230740/table: creating writer at 1731486534296 (+13 ms)Flushing 1588230740/table: appending metadata at 1731486534310 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731486534310Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f32a75a: reopening flushed file at 1731486534324 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1451c39d: reopening flushed file at 1731486534330 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25d4a58d: reopening flushed file at 1731486534336 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false at 1731486534343 (+7 ms)Writing region close event to WAL at 1731486534344 (+1 ms)Running coprocessor post-close hooks at 1731486534348 (+4 ms)Closed at 1731486534348 2024-11-13T08:28:54,349 DEBUG [RS_CLOSE_META-regionserver/dd3a0865bb23:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T08:28:54,431 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(976): stopping server dd3a0865bb23,35727,1731486497292; all regions closed. 2024-11-13T08:28:54,432 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(976): stopping server dd3a0865bb23,45659,1731486497432; all regions closed. 2024-11-13T08:28:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741835_1011 (size=3433) 2024-11-13T08:28:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741835_1011 (size=3433) 2024-11-13T08:28:54,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741835_1011 (size=3433) 2024-11-13T08:28:54,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741836_1012 (size=8899) 2024-11-13T08:28:54,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741836_1012 (size=8899) 2024-11-13T08:28:54,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741836_1012 (size=8899) 2024-11-13T08:28:54,436 DEBUG [RS:0;dd3a0865bb23:35727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs 2024-11-13T08:28:54,436 INFO [RS:0;dd3a0865bb23:35727 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd3a0865bb23%2C35727%2C1731486497292:(num 1731486498760) 2024-11-13T08:28:54,436 DEBUG [RS:0;dd3a0865bb23:35727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,436 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.ChoreService(370): Chore service for: regionserver/dd3a0865bb23:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T08:28:54,437 DEBUG [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T08:28:54,437 INFO [regionserver/dd3a0865bb23:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T08:28:54,437 INFO [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd3a0865bb23%2C45659%2C1731486497432.meta:.meta(num 1731486499120) 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T08:28:54,437 INFO [RS:0;dd3a0865bb23:35727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35727 2024-11-13T08:28:54,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd3a0865bb23,35727,1731486497292 2024-11-13T08:28:54,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T08:28:54,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741833_1009 (size=93) 2024-11-13T08:28:54,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741833_1009 (size=93) 2024-11-13T08:28:54,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741833_1009 (size=93) 2024-11-13T08:28:54,441 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd3a0865bb23,35727,1731486497292] 2024-11-13T08:28:54,443 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd3a0865bb23,35727,1731486497292 already deleted, retry=false 2024-11-13T08:28:54,443 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd3a0865bb23,35727,1731486497292 expired; onlineServers=1 2024-11-13T08:28:54,443 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T08:28:54,443 DEBUG [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/oldWALs 2024-11-13T08:28:54,443 INFO [RS:2;dd3a0865bb23:45659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd3a0865bb23%2C45659%2C1731486497432:(num 1731486498760) 2024-11-13T08:28:54,443 DEBUG [RS:2;dd3a0865bb23:45659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T08:28:54,443 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T08:28:54,443 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T08:28:54,444 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.ChoreService(370): Chore service for: regionserver/dd3a0865bb23:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T08:28:54,444 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T08:28:54,444 INFO [regionserver/dd3a0865bb23:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T08:28:54,444 INFO [RS:2;dd3a0865bb23:45659 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45659 2024-11-13T08:28:54,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T08:28:54,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd3a0865bb23,45659,1731486497432 2024-11-13T08:28:54,450 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T08:28:54,452 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd3a0865bb23,45659,1731486497432] 2024-11-13T08:28:54,453 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd3a0865bb23,45659,1731486497432 already deleted, retry=false 2024-11-13T08:28:54,453 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd3a0865bb23,45659,1731486497432 expired; onlineServers=0 2024-11-13T08:28:54,453 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd3a0865bb23,37177,1731486496596' ***** 2024-11-13T08:28:54,453 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T08:28:54,454 INFO [M:0;dd3a0865bb23:37177 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T08:28:54,454 INFO [M:0;dd3a0865bb23:37177 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T08:28:54,454 DEBUG [M:0;dd3a0865bb23:37177 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T08:28:54,454 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T08:28:54,454 DEBUG [M:0;dd3a0865bb23:37177 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T08:28:54,454 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster-HFileCleaner.small.0-1731486498381 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd3a0865bb23:0:becomeActiveMaster-HFileCleaner.small.0-1731486498381,5,FailOnTimeoutGroup] 2024-11-13T08:28:54,454 DEBUG [master/dd3a0865bb23:0:becomeActiveMaster-HFileCleaner.large.0-1731486498379 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd3a0865bb23:0:becomeActiveMaster-HFileCleaner.large.0-1731486498379,5,FailOnTimeoutGroup] 2024-11-13T08:28:54,454 INFO [M:0;dd3a0865bb23:37177 {}] hbase.ChoreService(370): Chore service for: master/dd3a0865bb23:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T08:28:54,454 INFO [M:0;dd3a0865bb23:37177 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T08:28:54,454 DEBUG [M:0;dd3a0865bb23:37177 {}] master.HMaster(1795): Stopping service threads 2024-11-13T08:28:54,454 INFO [M:0;dd3a0865bb23:37177 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T08:28:54,455 INFO [M:0;dd3a0865bb23:37177 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T08:28:54,455 INFO [M:0;dd3a0865bb23:37177 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T08:28:54,455 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T08:28:54,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T08:28:54,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T08:28:54,458 DEBUG [M:0;dd3a0865bb23:37177 {}] zookeeper.ZKUtil(347): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T08:28:54,458 WARN [M:0;dd3a0865bb23:37177 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T08:28:54,459 INFO [M:0;dd3a0865bb23:37177 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/.lastflushedseqids 2024-11-13T08:28:54,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741941_1121 (size=138) 2024-11-13T08:28:54,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741941_1121 (size=138) 2024-11-13T08:28:54,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741941_1121 (size=138) 2024-11-13T08:28:54,471 INFO [M:0;dd3a0865bb23:37177 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T08:28:54,472 INFO [M:0;dd3a0865bb23:37177 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T08:28:54,472 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T08:28:54,472 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:54,472 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:54,472 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T08:28:54,472 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:54,472 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-13T08:28:54,488 DEBUG [M:0;dd3a0865bb23:37177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee5f5ed3189949aba117638656cd940c is 82, key is hbase:meta,,1/info:regioninfo/1731486499200/Put/seqid=0 2024-11-13T08:28:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741942_1122 (size=5672) 2024-11-13T08:28:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741942_1122 (size=5672) 2024-11-13T08:28:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741942_1122 (size=5672) 2024-11-13T08:28:54,495 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee5f5ed3189949aba117638656cd940c 2024-11-13T08:28:54,514 DEBUG [M:0;dd3a0865bb23:37177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f394dc1e9a264b96a57c78c74ab167db is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731486511680/Put/seqid=0 2024-11-13T08:28:54,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741943_1123 (size=7755) 2024-11-13T08:28:54,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741943_1123 (size=7755) 2024-11-13T08:28:54,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741943_1123 (size=7755) 2024-11-13T08:28:54,524 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f394dc1e9a264b96a57c78c74ab167db 2024-11-13T08:28:54,529 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f394dc1e9a264b96a57c78c74ab167db 2024-11-13T08:28:54,544 DEBUG [M:0;dd3a0865bb23:37177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/844c422298dc4791806174b381a087be is 69, key is dd3a0865bb23,35727,1731486497292/rs:state/1731486498526/Put/seqid=0 2024-11-13T08:28:54,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:54,550 INFO [RS:0;dd3a0865bb23:35727 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T08:28:54,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35727-0x10110711ec40001, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:54,550 INFO [RS:0;dd3a0865bb23:35727 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd3a0865bb23,35727,1731486497292; zookeeper connection closed. 2024-11-13T08:28:54,550 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@343dcaf2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@343dcaf2 2024-11-13T08:28:54,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741944_1124 (size=5445) 2024-11-13T08:28:54,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741944_1124 (size=5445) 2024-11-13T08:28:54,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741944_1124 (size=5445) 2024-11-13T08:28:54,552 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/844c422298dc4791806174b381a087be 2024-11-13T08:28:54,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:54,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45659-0x10110711ec40003, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:54,552 INFO [RS:2;dd3a0865bb23:45659 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T08:28:54,552 INFO [RS:2;dd3a0865bb23:45659 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd3a0865bb23,45659,1731486497432; zookeeper connection closed. 2024-11-13T08:28:54,552 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5df9d7be {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5df9d7be 2024-11-13T08:28:54,553 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T08:28:54,557 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 844c422298dc4791806174b381a087be 2024-11-13T08:28:54,558 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee5f5ed3189949aba117638656cd940c as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ee5f5ed3189949aba117638656cd940c 2024-11-13T08:28:54,563 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ee5f5ed3189949aba117638656cd940c, entries=8, sequenceid=168, filesize=5.5 K 2024-11-13T08:28:54,564 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f394dc1e9a264b96a57c78c74ab167db as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f394dc1e9a264b96a57c78c74ab167db 2024-11-13T08:28:54,569 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f394dc1e9a264b96a57c78c74ab167db 2024-11-13T08:28:54,569 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f394dc1e9a264b96a57c78c74ab167db, entries=17, sequenceid=168, filesize=7.6 K 2024-11-13T08:28:54,570 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/844c422298dc4791806174b381a087be as hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/844c422298dc4791806174b381a087be 2024-11-13T08:28:54,574 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 844c422298dc4791806174b381a087be 2024-11-13T08:28:54,575 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38569/user/jenkins/test-data/5bb9e004-cee8-1c2d-0d74-586df926dbcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/844c422298dc4791806174b381a087be, entries=3, sequenceid=168, filesize=5.3 K 2024-11-13T08:28:54,576 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false 2024-11-13T08:28:54,577 INFO [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T08:28:54,577 DEBUG [M:0;dd3a0865bb23:37177 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731486534472Disabling compacts and flushes for region at 1731486534472Disabling writes for close at 1731486534472Obtaining lock to block concurrent updates at 1731486534472Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731486534472Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731486534473 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731486534473Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731486534473Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731486534487 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731486534487Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731486534499 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731486534513 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731486534513Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731486534529 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731486534544 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731486534544Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@551b4e51: reopening flushed file at 1731486534557 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6df9f0d8: reopening flushed file at 1731486534563 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e96392: reopening flushed file at 1731486534569 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false at 1731486534576 (+7 ms)Writing region close event to WAL at 1731486534577 (+1 ms)Closed at 1731486534577 2024-11-13T08:28:54,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34115 is added to blk_1073741830_1006 (size=81431) 2024-11-13T08:28:54,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44221 is added to blk_1073741830_1006 (size=81431) 2024-11-13T08:28:54,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34057 is added to blk_1073741830_1006 (size=81431) 2024-11-13T08:28:54,581 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T08:28:54,581 INFO [M:0;dd3a0865bb23:37177 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T08:28:54,582 INFO [M:0;dd3a0865bb23:37177 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37177 2024-11-13T08:28:54,582 INFO [M:0;dd3a0865bb23:37177 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T08:28:54,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:54,684 INFO [M:0;dd3a0865bb23:37177 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T08:28:54,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37177-0x10110711ec40000, quorum=127.0.0.1:62074, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T08:28:54,691 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514999 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514999 (inode 16655) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731486514690/wal.1731486514999 (inode 16655) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T08:28:54,692 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731486506813/wal.1731486506926 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:54,692 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731486529642/wal.1731486529829 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:54,695 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486524373 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486524373 (inode 16678) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731486515135/wal.1731486524373 (inode 16678) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T08:28:54,695 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731486500309/wal.1731486500379 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:54,697 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524976 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524976 (inode 16704) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731486524563/wal.1731486524976 (inode 16704) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T08:28:54,698 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731486525129/wal.1731486525194 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T08:28:54,699 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal.1731486500181 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal.1731486500181 (inode 16485) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731486499989/wal.1731486500181 (inode 16485) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T08:28:54,702 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal.1731486499697 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal.1731486499697 (inode 16462) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731486499465/wal.1731486499697 (inode 16462) Holder DFSClient_NONMAPREDUCE_-2133564029_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T08:28:54,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T08:28:54,708 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T08:28:54,708 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T08:28:54,708 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T08:28:54,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,STOPPED} 2024-11-13T08:28:54,711 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T08:28:54,711 WARN [BP-433341908-172.17.0.2-1731486493558 heartbeating to localhost/127.0.0.1:38569 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T08:28:54,711 WARN [BP-433341908-172.17.0.2-1731486493558 heartbeating to localhost/127.0.0.1:38569 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-433341908-172.17.0.2-1731486493558 (Datanode Uuid f421a173-ef64-4f0d-bddb-2955923dedc1) service to localhost/127.0.0.1:38569 2024-11-13T08:28:54,711 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T08:28:54,713 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data5/current/BP-433341908-172.17.0.2-1731486493558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T08:28:54,713 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data6/current/BP-433341908-172.17.0.2-1731486493558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T08:28:54,714 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T08:28:54,715 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T08:28:54,716 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T08:28:54,716 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T08:28:54,716 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T08:28:54,716 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,STOPPED} 2024-11-13T08:28:54,717 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T08:28:54,717 WARN [BP-433341908-172.17.0.2-1731486493558 heartbeating to localhost/127.0.0.1:38569 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T08:28:54,717 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T08:28:54,717 WARN [BP-433341908-172.17.0.2-1731486493558 heartbeating to localhost/127.0.0.1:38569 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-433341908-172.17.0.2-1731486493558 (Datanode Uuid 14c44a06-c44b-494c-a5eb-ac0c550a074b) service to localhost/127.0.0.1:38569 2024-11-13T08:28:54,718 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data3/current/BP-433341908-172.17.0.2-1731486493558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T08:28:54,718 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data4/current/BP-433341908-172.17.0.2-1731486493558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T08:28:54,718 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T08:28:54,724 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T08:28:54,725 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T08:28:54,725 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T08:28:54,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T08:28:54,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,STOPPED} 2024-11-13T08:28:54,726 WARN [BP-433341908-172.17.0.2-1731486493558 heartbeating to localhost/127.0.0.1:38569 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T08:28:54,726 WARN [BP-433341908-172.17.0.2-1731486493558 heartbeating to localhost/127.0.0.1:38569 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-433341908-172.17.0.2-1731486493558 (Datanode Uuid 2e7281ce-1c99-4aae-81ad-031d5d06ac21) service to localhost/127.0.0.1:38569 2024-11-13T08:28:54,726 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T08:28:54,726 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T08:28:54,727 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data1/current/BP-433341908-172.17.0.2-1731486493558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T08:28:54,727 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/cluster_06b5f094-6e82-d057-10da-ec559aec3ddd/data/data2/current/BP-433341908-172.17.0.2-1731486493558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T08:28:54,728 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T08:28:54,734 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T08:28:54,735 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T08:28:54,735 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T08:28:54,735 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T08:28:54,735 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c3c8d1db-b842-2d1e-055f-2a49a0f4a31f/hadoop.log.dir/,STOPPED} 2024-11-13T08:28:54,745 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T08:28:54,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down