2024-12-09 14:22:03,924 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 14:22:03,957 main DEBUG Took 0.030738 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 14:22:03,957 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 14:22:03,958 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 14:22:03,959 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 14:22:03,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:03,973 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 14:22:04,004 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,010 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,011 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,013 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,014 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,015 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,017 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,018 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,019 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,020 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,021 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:22:04,023 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,024 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 14:22:04,027 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:22:04,029 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 14:22:04,031 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 14:22:04,032 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 14:22:04,033 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 14:22:04,034 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 14:22:04,045 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 14:22:04,049 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 14:22:04,051 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 14:22:04,051 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 14:22:04,051 main DEBUG createAppenders(={Console}) 2024-12-09 14:22:04,052 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-09 14:22:04,052 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 14:22:04,053 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-09 14:22:04,053 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 14:22:04,053 main DEBUG OutputStream closed 2024-12-09 14:22:04,053 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 14:22:04,054 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 14:22:04,054 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-09 14:22:04,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 14:22:04,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 14:22:04,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 14:22:04,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 14:22:04,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 14:22:04,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 14:22:04,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 14:22:04,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 14:22:04,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 14:22:04,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 14:22:04,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 14:22:04,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 14:22:04,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 14:22:04,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 14:22:04,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 14:22:04,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 14:22:04,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 14:22:04,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 14:22:04,153 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 14:22:04,153 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-09 14:22:04,154 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 14:22:04,154 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-09T14:22:04,403 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530 2024-12-09 14:22:04,406 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 14:22:04,407 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T14:22:04,416 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T14:22:04,423 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-12-09T14:22:04,451 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T14:22:04,520 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T14:22:04,520 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T14:22:04,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T14:22:04,556 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d, deleteOnExit=true 2024-12-09T14:22:04,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T14:22:04,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/test.cache.data in system properties and HBase conf 2024-12-09T14:22:04,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T14:22:04,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir in system properties and HBase conf 2024-12-09T14:22:04,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T14:22:04,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T14:22:04,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T14:22:04,670 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T14:22:04,798 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:22:04,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T14:22:04,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T14:22:04,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T14:22:04,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T14:22:04,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T14:22:04,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T14:22:04,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T14:22:04,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T14:22:04,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T14:22:04,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/nfs.dump.dir in system properties and HBase conf 2024-12-09T14:22:04,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/java.io.tmpdir in system properties and HBase conf 2024-12-09T14:22:04,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T14:22:04,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T14:22:04,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T14:22:05,842 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T14:22:05,948 INFO [Time-limited test {}] log.Log(170): Logging initialized @2883ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T14:22:06,058 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:22:06,149 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:22:06,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:22:06,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:22:06,200 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T14:22:06,232 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:22:06,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:22:06,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:22:06,504 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c8b857b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/java.io.tmpdir/jetty-localhost-35081-hadoop-hdfs-3_4_1-tests_jar-_-any-6699699285094177817/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T14:22:06,512 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:35081} 2024-12-09T14:22:06,513 INFO [Time-limited test {}] server.Server(415): Started @3448ms 2024-12-09T14:22:07,005 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:22:07,013 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:22:07,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:22:07,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:22:07,018 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T14:22:07,019 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:22:07,020 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:22:07,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17d1a8d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/java.io.tmpdir/jetty-localhost-43179-hadoop-hdfs-3_4_1-tests_jar-_-any-1321540623234394448/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:22:07,143 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:43179} 2024-12-09T14:22:07,143 INFO [Time-limited test {}] server.Server(415): Started @4079ms 2024-12-09T14:22:07,200 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T14:22:07,340 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:22:07,353 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:22:07,371 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:22:07,371 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:22:07,371 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T14:22:07,372 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:22:07,373 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:22:07,525 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4464725f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/java.io.tmpdir/jetty-localhost-37925-hadoop-hdfs-3_4_1-tests_jar-_-any-9729562331389021537/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:22:07,525 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:37925} 2024-12-09T14:22:07,526 INFO [Time-limited test {}] server.Server(415): Started @4462ms 2024-12-09T14:22:07,529 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T14:22:07,585 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:22:07,591 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:22:07,595 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:22:07,596 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:22:07,596 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T14:22:07,599 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:22:07,601 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:22:07,718 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data2/current/BP-728790427-172.17.0.3-1733754125568/current, will proceed with Du for space computation calculation, 2024-12-09T14:22:07,718 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data4/current/BP-728790427-172.17.0.3-1733754125568/current, will proceed with Du for space computation calculation, 2024-12-09T14:22:07,719 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data1/current/BP-728790427-172.17.0.3-1733754125568/current, will proceed with Du for space computation calculation, 2024-12-09T14:22:07,719 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data3/current/BP-728790427-172.17.0.3-1733754125568/current, will proceed with Du for space computation calculation, 2024-12-09T14:22:07,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26f8cb79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/java.io.tmpdir/jetty-localhost-34277-hadoop-hdfs-3_4_1-tests_jar-_-any-11547735143823369766/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:22:07,768 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:34277} 2024-12-09T14:22:07,768 INFO [Time-limited test {}] server.Server(415): Started @4704ms 2024-12-09T14:22:07,771 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T14:22:07,814 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T14:22:07,814 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T14:22:07,905 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a058fdfe1642d36 with lease ID 0xc1e187cf2b15a614: Processing first storage report for DS-a53f40d1-818e-413a-b669-5a24c58c008c from datanode DatanodeRegistration(127.0.0.1:34887, datanodeUuid=d475c1ca-f353-47c5-aa02-45e4cda55273, infoPort=44507, infoSecurePort=0, ipcPort=34121, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568) 2024-12-09T14:22:07,910 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a058fdfe1642d36 with lease ID 0xc1e187cf2b15a614: from storage DS-a53f40d1-818e-413a-b669-5a24c58c008c node DatanodeRegistration(127.0.0.1:34887, datanodeUuid=d475c1ca-f353-47c5-aa02-45e4cda55273, infoPort=44507, infoSecurePort=0, ipcPort=34121, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-12-09T14:22:07,910 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6d53a79a4fde242f with lease ID 0xc1e187cf2b15a613: Processing first storage report for DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83 from datanode DatanodeRegistration(127.0.0.1:34769, datanodeUuid=0ae23200-61d1-484e-8543-bc293776a38b, infoPort=41675, infoSecurePort=0, ipcPort=38855, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568) 2024-12-09T14:22:07,911 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6d53a79a4fde242f with lease ID 0xc1e187cf2b15a613: from storage DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83 node DatanodeRegistration(127.0.0.1:34769, datanodeUuid=0ae23200-61d1-484e-8543-bc293776a38b, infoPort=41675, infoSecurePort=0, ipcPort=38855, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T14:22:07,912 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6d53a79a4fde242f with lease ID 0xc1e187cf2b15a613: Processing first storage report for DS-f588e49b-3e8b-4c74-bfb0-2b00358a166b from datanode DatanodeRegistration(127.0.0.1:34769, datanodeUuid=0ae23200-61d1-484e-8543-bc293776a38b, infoPort=41675, infoSecurePort=0, ipcPort=38855, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568) 2024-12-09T14:22:07,912 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6d53a79a4fde242f with lease ID 0xc1e187cf2b15a613: from storage DS-f588e49b-3e8b-4c74-bfb0-2b00358a166b node DatanodeRegistration(127.0.0.1:34769, datanodeUuid=0ae23200-61d1-484e-8543-bc293776a38b, infoPort=41675, infoSecurePort=0, ipcPort=38855, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T14:22:07,914 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a058fdfe1642d36 with lease ID 0xc1e187cf2b15a614: Processing first storage report for DS-96356b9f-e895-4749-b796-df6d6f9bd440 from datanode DatanodeRegistration(127.0.0.1:34887, datanodeUuid=d475c1ca-f353-47c5-aa02-45e4cda55273, infoPort=44507, infoSecurePort=0, ipcPort=34121, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568) 2024-12-09T14:22:07,914 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a058fdfe1642d36 with lease ID 0xc1e187cf2b15a614: from storage DS-96356b9f-e895-4749-b796-df6d6f9bd440 node DatanodeRegistration(127.0.0.1:34887, datanodeUuid=d475c1ca-f353-47c5-aa02-45e4cda55273, infoPort=44507, infoSecurePort=0, ipcPort=34121, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T14:22:08,001 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data5/current/BP-728790427-172.17.0.3-1733754125568/current, will proceed with Du for space computation calculation, 2024-12-09T14:22:08,009 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data6/current/BP-728790427-172.17.0.3-1733754125568/current, will proceed with Du for space computation calculation, 2024-12-09T14:22:08,070 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T14:22:08,080 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x673518651ba8849b with lease ID 0xc1e187cf2b15a615: Processing first storage report for DS-5da0b2d7-a79e-4338-832f-f553d838b0d6 from datanode DatanodeRegistration(127.0.0.1:35501, datanodeUuid=5db1e07e-9907-45e2-931e-a86e2b84132d, infoPort=33273, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568) 2024-12-09T14:22:08,080 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x673518651ba8849b with lease ID 0xc1e187cf2b15a615: from storage DS-5da0b2d7-a79e-4338-832f-f553d838b0d6 node DatanodeRegistration(127.0.0.1:35501, datanodeUuid=5db1e07e-9907-45e2-931e-a86e2b84132d, infoPort=33273, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T14:22:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x673518651ba8849b with lease ID 0xc1e187cf2b15a615: Processing first storage report for DS-085a0579-5aab-4db9-adaf-7ccd43171ed7 from datanode DatanodeRegistration(127.0.0.1:35501, datanodeUuid=5db1e07e-9907-45e2-931e-a86e2b84132d, infoPort=33273, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568) 2024-12-09T14:22:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x673518651ba8849b with lease ID 0xc1e187cf2b15a615: from storage DS-085a0579-5aab-4db9-adaf-7ccd43171ed7 node DatanodeRegistration(127.0.0.1:35501, datanodeUuid=5db1e07e-9907-45e2-931e-a86e2b84132d, infoPort=33273, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=93058663;c=1733754125568), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T14:22:08,350 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530 2024-12-09T14:22:08,478 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/zookeeper_0, clientPort=49681, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T14:22:08,522 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49681 2024-12-09T14:22:08,540 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:08,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:08,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741825_1001 (size=7) 2024-12-09T14:22:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741825_1001 (size=7) 2024-12-09T14:22:08,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741825_1001 (size=7) 2024-12-09T14:22:09,164 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9 with version=8 2024-12-09T14:22:09,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/hbase-staging 2024-12-09T14:22:09,480 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f4e784dc7cb5:0 server-side Connection retries=45 2024-12-09T14:22:09,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:09,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:09,504 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:22:09,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:09,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:22:09,689 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T14:22:09,769 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T14:22:09,779 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T14:22:09,784 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:22:09,816 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17853 (auto-detected) 2024-12-09T14:22:09,818 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T14:22:09,849 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39231 2024-12-09T14:22:09,881 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39231 connecting to ZooKeeper ensemble=127.0.0.1:49681 2024-12-09T14:22:09,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:392310x0, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:22:09,956 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39231-0x1012b90185b0000 connected 2024-12-09T14:22:10,047 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,052 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,067 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:10,072 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9, hbase.cluster.distributed=false 2024-12-09T14:22:10,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:22:10,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39231 2024-12-09T14:22:10,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39231 2024-12-09T14:22:10,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39231 2024-12-09T14:22:10,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39231 2024-12-09T14:22:10,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39231 2024-12-09T14:22:10,263 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f4e784dc7cb5:0 server-side Connection retries=45 2024-12-09T14:22:10,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,266 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:22:10,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:22:10,270 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:22:10,273 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:22:10,274 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33819 2024-12-09T14:22:10,276 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33819 connecting to ZooKeeper ensemble=127.0.0.1:49681 2024-12-09T14:22:10,277 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338190x0, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:22:10,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:338190x0, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:10,288 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33819-0x1012b90185b0001 connected 2024-12-09T14:22:10,293 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:22:10,303 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:22:10,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:22:10,311 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:22:10,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33819 2024-12-09T14:22:10,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33819 2024-12-09T14:22:10,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33819 2024-12-09T14:22:10,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33819 2024-12-09T14:22:10,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33819 2024-12-09T14:22:10,338 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f4e784dc7cb5:0 server-side Connection retries=45 2024-12-09T14:22:10,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,339 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:22:10,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,340 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:22:10,340 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:22:10,340 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:22:10,341 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43199 2024-12-09T14:22:10,343 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43199 connecting to ZooKeeper ensemble=127.0.0.1:49681 2024-12-09T14:22:10,344 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,346 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431990x0, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:22:10,353 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43199-0x1012b90185b0002 connected 2024-12-09T14:22:10,357 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:10,358 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:22:10,359 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:22:10,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:22:10,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:22:10,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43199 2024-12-09T14:22:10,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43199 2024-12-09T14:22:10,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43199 2024-12-09T14:22:10,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43199 2024-12-09T14:22:10,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43199 2024-12-09T14:22:10,392 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f4e784dc7cb5:0 server-side Connection retries=45 2024-12-09T14:22:10,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,393 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:22:10,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:22:10,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:22:10,394 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:22:10,394 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:22:10,396 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40373 2024-12-09T14:22:10,397 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40373 connecting to ZooKeeper ensemble=127.0.0.1:49681 2024-12-09T14:22:10,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,402 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403730x0, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:22:10,410 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:403730x0, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:10,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40373-0x1012b90185b0003 connected 2024-12-09T14:22:10,411 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:22:10,414 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:22:10,416 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:22:10,419 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:22:10,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40373 2024-12-09T14:22:10,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40373 2024-12-09T14:22:10,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40373 2024-12-09T14:22:10,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40373 2024-12-09T14:22:10,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40373 2024-12-09T14:22:10,452 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f4e784dc7cb5:39231 2024-12-09T14:22:10,454 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:10,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,471 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:22:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:22:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:22:10,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,502 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T14:22:10,507 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f4e784dc7cb5,39231,1733754129274 from backup master directory 2024-12-09T14:22:10,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:10,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:22:10,515 WARN [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:22:10,515 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:10,518 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T14:22:10,522 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T14:22:10,589 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/hbase.id] with ID: 1a8cc862-cf33-438f-b5cf-d9e9d16321ec 2024-12-09T14:22:10,590 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/.tmp/hbase.id 2024-12-09T14:22:10,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741826_1002 (size=42) 2024-12-09T14:22:10,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741826_1002 (size=42) 2024-12-09T14:22:10,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741826_1002 (size=42) 2024-12-09T14:22:10,609 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/.tmp/hbase.id]:[hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/hbase.id] 2024-12-09T14:22:10,656 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:22:10,661 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T14:22:10,680 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-09T14:22:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:10,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741827_1003 (size=196) 2024-12-09T14:22:10,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741827_1003 (size=196) 2024-12-09T14:22:10,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741827_1003 (size=196) 2024-12-09T14:22:10,740 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:22:10,742 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T14:22:10,751 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:10,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741828_1004 (size=1189) 2024-12-09T14:22:10,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741828_1004 (size=1189) 2024-12-09T14:22:10,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741828_1004 (size=1189) 2024-12-09T14:22:10,814 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store 2024-12-09T14:22:10,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741829_1005 (size=34) 2024-12-09T14:22:10,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741829_1005 (size=34) 2024-12-09T14:22:10,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741829_1005 (size=34) 2024-12-09T14:22:10,842 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T14:22:10,845 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:10,846 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:22:10,846 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:10,846 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:10,848 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:22:10,848 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:10,848 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:10,849 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733754130846Disabling compacts and flushes for region at 1733754130846Disabling writes for close at 1733754130848 (+2 ms)Writing region close event to WAL at 1733754130848Closed at 1733754130848 2024-12-09T14:22:10,851 WARN [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/.initializing 2024-12-09T14:22:10,851 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/WALs/f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:10,859 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:10,873 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f4e784dc7cb5%2C39231%2C1733754129274, suffix=, logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/WALs/f4e784dc7cb5,39231,1733754129274, archiveDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/oldWALs, maxLogs=10 2024-12-09T14:22:10,906 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/WALs/f4e784dc7cb5,39231,1733754129274/f4e784dc7cb5%2C39231%2C1733754129274.1733754130878, exclude list is [], retry=0 2024-12-09T14:22:10,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:10,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:10,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:10,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:10,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T14:22:10,968 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/WALs/f4e784dc7cb5,39231,1733754129274/f4e784dc7cb5%2C39231%2C1733754129274.1733754130878 2024-12-09T14:22:10,969 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:10,970 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:10,970 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:10,973 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:10,974 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,010 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,036 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T14:22:11,040 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:11,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T14:22:11,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:11,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T14:22:11,050 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:11,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T14:22:11,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:11,054 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,057 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,058 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,063 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,064 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,067 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:22:11,070 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:22:11,074 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:11,075 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75045309, jitterRate=0.11826224625110626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:22:11,081 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733754130986Initializing all the Stores at 1733754130987 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754130988 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754130989 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754130989Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754130989Cleaning up temporary data from old regions at 1733754131064 (+75 ms)Region opened successfully at 1733754131081 (+17 ms) 2024-12-09T14:22:11,083 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T14:22:11,115 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e86fd8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f4e784dc7cb5/172.17.0.3:0 2024-12-09T14:22:11,147 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T14:22:11,157 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T14:22:11,158 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T14:22:11,161 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T14:22:11,162 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T14:22:11,166 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T14:22:11,167 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T14:22:11,192 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T14:22:11,200 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T14:22:11,204 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T14:22:11,207 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T14:22:11,209 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T14:22:11,211 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T14:22:11,213 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T14:22:11,217 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T14:22:11,219 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T14:22:11,220 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T14:22:11,222 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T14:22:11,241 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T14:22:11,243 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T14:22:11,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:11,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:11,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:11,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:11,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,251 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f4e784dc7cb5,39231,1733754129274, sessionid=0x1012b90185b0000, setting cluster-up flag (Was=false) 2024-12-09T14:22:11,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,270 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T14:22:11,272 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:11,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:11,284 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T14:22:11,286 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:11,293 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T14:22:11,339 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(746): ClusterId : 1a8cc862-cf33-438f-b5cf-d9e9d16321ec 2024-12-09T14:22:11,340 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(746): ClusterId : 1a8cc862-cf33-438f-b5cf-d9e9d16321ec 2024-12-09T14:22:11,340 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(746): ClusterId : 1a8cc862-cf33-438f-b5cf-d9e9d16321ec 2024-12-09T14:22:11,344 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:22:11,344 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:22:11,345 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:22:11,352 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:22:11,352 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:22:11,352 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:22:11,352 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:22:11,355 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:22:11,355 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:22:11,358 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:22:11,358 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:22:11,359 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:22:11,359 DEBUG [RS:1;f4e784dc7cb5:43199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a79283e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f4e784dc7cb5/172.17.0.3:0 2024-12-09T14:22:11,360 DEBUG [RS:2;f4e784dc7cb5:40373 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b046136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f4e784dc7cb5/172.17.0.3:0 2024-12-09T14:22:11,360 DEBUG [RS:0;f4e784dc7cb5:33819 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f6c3ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f4e784dc7cb5/172.17.0.3:0 2024-12-09T14:22:11,380 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f4e784dc7cb5:43199 2024-12-09T14:22:11,385 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:22:11,385 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:22:11,385 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:22:11,388 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(2659): reportForDuty to master=f4e784dc7cb5,39231,1733754129274 with port=43199, startcode=1733754130338 2024-12-09T14:22:11,394 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f4e784dc7cb5:40373 2024-12-09T14:22:11,394 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:22:11,394 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:22:11,394 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:22:11,396 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(2659): reportForDuty to master=f4e784dc7cb5,39231,1733754129274 with port=40373, startcode=1733754130392 2024-12-09T14:22:11,399 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f4e784dc7cb5:33819 2024-12-09T14:22:11,399 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:22:11,399 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:22:11,399 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:22:11,401 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(2659): reportForDuty to master=f4e784dc7cb5,39231,1733754129274 with port=33819, startcode=1733754130225 2024-12-09T14:22:11,407 DEBUG [RS:1;f4e784dc7cb5:43199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:22:11,407 DEBUG [RS:0;f4e784dc7cb5:33819 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:22:11,407 DEBUG [RS:2;f4e784dc7cb5:40373 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:22:11,418 INFO [AsyncFSWAL-0-hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData-prefix:f4e784dc7cb5,39231,1733754129274 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-12-09T14:22:11,452 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T14:22:11,459 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35889, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:22:11,460 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46149, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:22:11,462 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33669, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:22:11,466 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T14:22:11,466 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T14:22:11,472 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T14:22:11,473 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T14:22:11,474 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T14:22:11,479 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f4e784dc7cb5,39231,1733754129274 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T14:22:11,489 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f4e784dc7cb5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:22:11,489 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f4e784dc7cb5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:22:11,489 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f4e784dc7cb5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:22:11,489 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f4e784dc7cb5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:22:11,490 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f4e784dc7cb5:0, corePoolSize=10, maxPoolSize=10 2024-12-09T14:22:11,490 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,490 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f4e784dc7cb5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:22:11,490 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,500 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733754161500 2024-12-09T14:22:11,501 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T14:22:11,501 WARN [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T14:22:11,502 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T14:22:11,502 WARN [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T14:22:11,502 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T14:22:11,503 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T14:22:11,503 WARN [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T14:22:11,503 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T14:22:11,507 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:22:11,508 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T14:22:11,508 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T14:22:11,508 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T14:22:11,508 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T14:22:11,509 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T14:22:11,510 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,515 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T14:22:11,517 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T14:22:11,517 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,517 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T14:22:11,517 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T14:22:11,520 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T14:22:11,520 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T14:22:11,524 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:11,524 WARN [IPC Server handler 2 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:11,524 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:11,527 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f4e784dc7cb5:0:becomeActiveMaster-HFileCleaner.large.0-1733754131521,5,FailOnTimeoutGroup] 2024-12-09T14:22:11,534 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f4e784dc7cb5:0:becomeActiveMaster-HFileCleaner.small.0-1733754131527,5,FailOnTimeoutGroup] 2024-12-09T14:22:11,535 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,535 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T14:22:11,536 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,537 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741831_1007 (size=1321) 2024-12-09T14:22:11,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741831_1007 (size=1321) 2024-12-09T14:22:11,572 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T14:22:11,573 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9 2024-12-09T14:22:11,577 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:11,577 WARN [IPC Server handler 0 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:11,577 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:11,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741832_1008 (size=32) 2024-12-09T14:22:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741832_1008 (size=32) 2024-12-09T14:22:11,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:11,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T14:22:11,591 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:22:11,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:11,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T14:22:11,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:22:11,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:11,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T14:22:11,599 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:22:11,599 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:11,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T14:22:11,603 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(2659): reportForDuty to master=f4e784dc7cb5,39231,1733754129274 with port=43199, startcode=1733754130338 2024-12-09T14:22:11,603 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(2659): reportForDuty to master=f4e784dc7cb5,39231,1733754129274 with port=40373, startcode=1733754130392 2024-12-09T14:22:11,604 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(2659): reportForDuty to master=f4e784dc7cb5,39231,1733754129274 with port=33819, startcode=1733754130225 2024-12-09T14:22:11,606 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:11,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:22:11,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:11,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:11,609 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(517): Registering regionserver=f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:11,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:22:11,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740 2024-12-09T14:22:11,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740 2024-12-09T14:22:11,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:22:11,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:22:11,618 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:22:11,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:22:11,623 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:11,623 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9 2024-12-09T14:22:11,623 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43873 2024-12-09T14:22:11,623 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(517): Registering regionserver=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:11,623 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:22:11,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:22:11,629 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9 2024-12-09T14:22:11,629 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43873 2024-12-09T14:22:11,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:11,629 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:22:11,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] master.ServerManager(517): Registering regionserver=f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:11,636 DEBUG [RS:1;f4e784dc7cb5:43199 {}] zookeeper.ZKUtil(111): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:11,636 WARN [RS:1;f4e784dc7cb5:43199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:22:11,636 INFO [RS:1;f4e784dc7cb5:43199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:11,637 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:11,637 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:11,637 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9 2024-12-09T14:22:11,637 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43873 2024-12-09T14:22:11,637 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:22:11,638 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70053236, jitterRate=0.04387456178665161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:22:11,638 DEBUG [RS:2;f4e784dc7cb5:40373 {}] zookeeper.ZKUtil(111): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:11,638 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f4e784dc7cb5,40373,1733754130392] 2024-12-09T14:22:11,638 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f4e784dc7cb5,43199,1733754130338] 2024-12-09T14:22:11,638 WARN [RS:2;f4e784dc7cb5:40373 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:22:11,639 INFO [RS:2;f4e784dc7cb5:40373 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:11,639 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:11,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733754131586Initializing all the Stores at 1733754131588 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754131588Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754131588Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754131588Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754131588Cleaning up temporary data from old regions at 1733754131616 (+28 ms)Region opened successfully at 1733754131641 (+25 ms) 2024-12-09T14:22:11,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:22:11,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:22:11,642 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:22:11,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:22:11,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:22:11,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:22:11,642 DEBUG [RS:0;f4e784dc7cb5:33819 {}] zookeeper.ZKUtil(111): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:11,642 WARN [RS:0;f4e784dc7cb5:33819 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:22:11,643 INFO [RS:0;f4e784dc7cb5:33819 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:11,643 DEBUG [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:11,643 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f4e784dc7cb5,33819,1733754130225] 2024-12-09T14:22:11,645 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:22:11,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733754131641Disabling compacts and flushes for region at 1733754131641Disabling writes for close at 1733754131642 (+1 ms)Writing region close event to WAL at 1733754131644 (+2 ms)Closed at 1733754131645 (+1 ms) 2024-12-09T14:22:11,648 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:22:11,649 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T14:22:11,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T14:22:11,667 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:22:11,670 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T14:22:11,672 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:22:11,672 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:22:11,673 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:22:11,690 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:22:11,690 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:22:11,690 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:22:11,697 INFO [RS:1;f4e784dc7cb5:43199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:22:11,697 INFO [RS:2;f4e784dc7cb5:40373 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:22:11,697 INFO [RS:0;f4e784dc7cb5:33819 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:22:11,697 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,697 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,697 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,698 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:22:11,698 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:22:11,702 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:22:11,706 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:22:11,706 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:22:11,708 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,708 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,708 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,708 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,708 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,708 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,708 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:22:11,709 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:22:11,709 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,709 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,710 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f4e784dc7cb5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:22:11,711 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f4e784dc7cb5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:22:11,711 DEBUG [RS:0;f4e784dc7cb5:33819 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f4e784dc7cb5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:22:11,711 DEBUG [RS:2;f4e784dc7cb5:40373 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f4e784dc7cb5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:22:11,713 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:22:11,713 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,714 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,714 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,714 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,714 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,715 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,715 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:22:11,715 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,715 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,715 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,715 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,716 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,716 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f4e784dc7cb5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:22:11,716 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f4e784dc7cb5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:22:11,716 DEBUG [RS:1;f4e784dc7cb5:43199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f4e784dc7cb5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:22:11,718 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,718 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,718 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,718 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,718 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,718 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,40373,1733754130392-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:22:11,754 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:22:11,757 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,40373,1733754130392-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,757 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,758 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.Replication(171): f4e784dc7cb5,40373,1733754130392 started 2024-12-09T14:22:11,764 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,43199,1733754130338-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:22:11,764 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,33819,1733754130225-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:22:11,786 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,789 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:22:11,789 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,43199,1733754130338-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,790 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:22:11,790 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,790 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,33819,1733754130225-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,790 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.Replication(171): f4e784dc7cb5,43199,1733754130338 started 2024-12-09T14:22:11,790 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,790 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.Replication(171): f4e784dc7cb5,33819,1733754130225 started 2024-12-09T14:22:11,793 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1482): Serving as f4e784dc7cb5,40373,1733754130392, RpcServer on f4e784dc7cb5/172.17.0.3:40373, sessionid=0x1012b90185b0003 2024-12-09T14:22:11,794 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:22:11,794 DEBUG [RS:2;f4e784dc7cb5:40373 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:11,794 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f4e784dc7cb5,40373,1733754130392' 2024-12-09T14:22:11,795 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:22:11,797 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:22:11,798 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:22:11,798 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:22:11,798 DEBUG [RS:2;f4e784dc7cb5:40373 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:11,799 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f4e784dc7cb5,40373,1733754130392' 2024-12-09T14:22:11,799 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:22:11,804 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:22:11,805 DEBUG [RS:2;f4e784dc7cb5:40373 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:22:11,805 INFO [RS:2;f4e784dc7cb5:40373 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:22:11,805 INFO [RS:2;f4e784dc7cb5:40373 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:22:11,813 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,813 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1482): Serving as f4e784dc7cb5,43199,1733754130338, RpcServer on f4e784dc7cb5/172.17.0.3:43199, sessionid=0x1012b90185b0002 2024-12-09T14:22:11,813 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:22:11,814 DEBUG [RS:1;f4e784dc7cb5:43199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:11,814 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f4e784dc7cb5,43199,1733754130338' 2024-12-09T14:22:11,814 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:22:11,814 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:11,814 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(1482): Serving as f4e784dc7cb5,33819,1733754130225, RpcServer on f4e784dc7cb5/172.17.0.3:33819, sessionid=0x1012b90185b0001 2024-12-09T14:22:11,814 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:22:11,814 DEBUG [RS:0;f4e784dc7cb5:33819 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:11,814 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f4e784dc7cb5,33819,1733754130225' 2024-12-09T14:22:11,815 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:22:11,815 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:22:11,815 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:22:11,816 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:22:11,816 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:22:11,816 DEBUG [RS:1;f4e784dc7cb5:43199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:11,816 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f4e784dc7cb5,43199,1733754130338' 2024-12-09T14:22:11,816 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:22:11,816 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:22:11,817 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:22:11,817 DEBUG [RS:0;f4e784dc7cb5:33819 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:11,817 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f4e784dc7cb5,33819,1733754130225' 2024-12-09T14:22:11,817 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:22:11,817 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:22:11,818 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:22:11,818 DEBUG [RS:1;f4e784dc7cb5:43199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:22:11,818 INFO [RS:1;f4e784dc7cb5:43199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:22:11,818 INFO [RS:1;f4e784dc7cb5:43199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:22:11,819 DEBUG [RS:0;f4e784dc7cb5:33819 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:22:11,819 INFO [RS:0;f4e784dc7cb5:33819 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:22:11,819 INFO [RS:0;f4e784dc7cb5:33819 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:22:11,821 WARN [f4e784dc7cb5:39231 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T14:22:11,910 INFO [RS:2;f4e784dc7cb5:40373 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:11,914 INFO [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f4e784dc7cb5%2C40373%2C1733754130392, suffix=, logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392, archiveDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs, maxLogs=32 2024-12-09T14:22:11,919 INFO [RS:1;f4e784dc7cb5:43199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:11,920 INFO [RS:0;f4e784dc7cb5:33819 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:11,922 INFO [RS:1;f4e784dc7cb5:43199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f4e784dc7cb5%2C43199%2C1733754130338, suffix=, logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338, archiveDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs, maxLogs=32 2024-12-09T14:22:11,923 INFO [RS:0;f4e784dc7cb5:33819 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f4e784dc7cb5%2C33819%2C1733754130225, suffix=, logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,33819,1733754130225, archiveDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs, maxLogs=32 2024-12-09T14:22:11,938 DEBUG [RS:2;f4e784dc7cb5:40373 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392/f4e784dc7cb5%2C40373%2C1733754130392.1733754131916, exclude list is [], retry=0 2024-12-09T14:22:11,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:11,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:11,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:11,948 DEBUG [RS:1;f4e784dc7cb5:43199 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, exclude list is [], retry=0 2024-12-09T14:22:11,951 DEBUG [RS:0;f4e784dc7cb5:33819 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,33819,1733754130225/f4e784dc7cb5%2C33819%2C1733754130225.1733754131925, exclude list is [], retry=0 2024-12-09T14:22:11,952 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:11,953 WARN [IPC Server handler 2 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:11,953 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:11,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:11,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:11,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:11,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:11,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:11,961 INFO [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392/f4e784dc7cb5%2C40373%2C1733754130392.1733754131916 2024-12-09T14:22:11,963 DEBUG [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:11,968 INFO [RS:1;f4e784dc7cb5:43199 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 2024-12-09T14:22:11,968 INFO [RS:0;f4e784dc7cb5:33819 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,33819,1733754130225/f4e784dc7cb5%2C33819%2C1733754130225.1733754131925 2024-12-09T14:22:11,968 DEBUG [RS:1;f4e784dc7cb5:43199 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:11,969 DEBUG [RS:0;f4e784dc7cb5:33819 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:12,074 DEBUG [f4e784dc7cb5:39231 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T14:22:12,083 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(204): Hosts are {f4e784dc7cb5=0} racks are {/default-rack=0} 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T14:22:12,090 INFO [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:22:12,090 INFO [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:22:12,090 INFO [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T14:22:12,090 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:22:12,097 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:12,103 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f4e784dc7cb5,40373,1733754130392, state=OPENING 2024-12-09T14:22:12,108 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T14:22:12,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:12,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:12,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:12,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:12,113 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,113 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,115 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:22:12,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f4e784dc7cb5,40373,1733754130392}] 2024-12-09T14:22:12,297 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:22:12,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46239, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:22:12,308 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T14:22:12,309 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:12,309 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T14:22:12,313 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f4e784dc7cb5%2C40373%2C1733754130392.meta, suffix=.meta, logDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392, archiveDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs, maxLogs=32 2024-12-09T14:22:12,334 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392/f4e784dc7cb5%2C40373%2C1733754130392.meta.1733754132315.meta, exclude list is [], retry=0 2024-12-09T14:22:12,336 WARN [IPC Server handler 4 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,337 WARN [IPC Server handler 4 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:12,337 WARN [IPC Server handler 4 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:12,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:12,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:12,342 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,40373,1733754130392/f4e784dc7cb5%2C40373%2C1733754130392.meta.1733754132315.meta 2024-12-09T14:22:12,343 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:12,343 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:12,345 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T14:22:12,347 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T14:22:12,352 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T14:22:12,357 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T14:22:12,357 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:12,358 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T14:22:12,358 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T14:22:12,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T14:22:12,363 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:22:12,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:12,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:12,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T14:22:12,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:22:12,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:12,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:12,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T14:22:12,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:22:12,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:12,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:12,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T14:22:12,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:22:12,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:12,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:22:12,371 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:22:12,372 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740 2024-12-09T14:22:12,375 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740 2024-12-09T14:22:12,378 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:22:12,378 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:22:12,379 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:22:12,382 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:22:12,384 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68102186, jitterRate=0.014801651239395142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:22:12,384 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T14:22:12,386 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733754132358Writing region info on filesystem at 1733754132358Initializing all the Stores at 1733754132361 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754132361Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754132361Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754132361Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733754132361Cleaning up temporary data from old regions at 1733754132378 (+17 ms)Running coprocessor post-open hooks at 1733754132384 (+6 ms)Region opened successfully at 1733754132386 (+2 ms) 2024-12-09T14:22:12,396 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733754132288 2024-12-09T14:22:12,410 DEBUG [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T14:22:12,411 INFO [RS_OPEN_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T14:22:12,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:12,423 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f4e784dc7cb5,40373,1733754130392, state=OPEN 2024-12-09T14:22:12,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:22:12,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:22:12,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:22:12,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,427 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,427 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:12,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:22:12,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:22:12,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T14:22:12,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f4e784dc7cb5,40373,1733754130392 in 310 msec 2024-12-09T14:22:12,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T14:22:12,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 780 msec 2024-12-09T14:22:12,445 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:22:12,445 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T14:22:12,467 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:22:12,469 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f4e784dc7cb5,40373,1733754130392, seqNum=-1] 2024-12-09T14:22:12,498 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:22:12,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39515, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:22:12,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1920 sec 2024-12-09T14:22:12,530 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733754132530, completionTime=-1 2024-12-09T14:22:12,560 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T14:22:12,560 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T14:22:12,597 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T14:22:12,597 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733754192597 2024-12-09T14:22:12,597 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733754252597 2024-12-09T14:22:12,598 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 37 msec 2024-12-09T14:22:12,600 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T14:22:12,616 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,39231,1733754129274-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,616 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,39231,1733754129274-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,616 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,39231,1733754129274-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,618 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f4e784dc7cb5:39231, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,619 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,619 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,626 DEBUG [master/f4e784dc7cb5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T14:22:12,656 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.140sec 2024-12-09T14:22:12,658 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T14:22:12,660 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T14:22:12,661 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T14:22:12,662 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T14:22:12,662 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T14:22:12,663 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,39231,1733754129274-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:22:12,663 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,39231,1733754129274-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T14:22:12,671 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T14:22:12,672 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T14:22:12,672 INFO [master/f4e784dc7cb5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f4e784dc7cb5,39231,1733754129274-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:12,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60cc2108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:22:12,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f4e784dc7cb5,39231,-1 for getting cluster id 2024-12-09T14:22:12,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T14:22:12,686 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1a8cc862-cf33-438f-b5cf-d9e9d16321ec' 2024-12-09T14:22:12,689 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T14:22:12,690 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1a8cc862-cf33-438f-b5cf-d9e9d16321ec" 2024-12-09T14:22:12,690 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dea6a5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:22:12,690 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f4e784dc7cb5,39231,-1] 2024-12-09T14:22:12,693 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T14:22:12,695 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:12,696 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T14:22:12,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9851f95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:22:12,701 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:22:12,711 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f4e784dc7cb5,40373,1733754130392, seqNum=-1] 2024-12-09T14:22:12,712 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:22:12,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:22:12,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:12,743 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43873/hbase 2024-12-09T14:22:12,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=359, OpenFileDescriptor=599, MaxFileDescriptor=1048576, SystemLoadAverage=553, ProcessCount=11, AvailableMemoryMB=6551 2024-12-09T14:22:12,785 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:12,789 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:12,790 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:12,795 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12852764, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-12852764, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:12,818 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12852764/hregion-12852764.1733754132796, exclude list is [], retry=0 2024-12-09T14:22:12,821 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,821 WARN [IPC Server handler 2 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:12,821 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:12,824 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:12,825 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:12,832 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12852764/hregion-12852764.1733754132796 2024-12-09T14:22:12,834 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:12,835 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1c47252ccb11fb63bd977e4f12f69260, NAME => 'testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:12,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741838_1014 (size=64) 2024-12-09T14:22:12,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741838_1014 (size=64) 2024-12-09T14:22:12,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741838_1014 (size=64) 2024-12-09T14:22:12,850 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:12,852 INFO [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,854 INFO [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c47252ccb11fb63bd977e4f12f69260 columnFamilyName a 2024-12-09T14:22:12,854 DEBUG [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:12,855 INFO [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] regionserver.HStore(327): Store=1c47252ccb11fb63bd977e4f12f69260/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:12,855 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,856 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,857 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,858 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,858 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,861 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:12,865 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:12,865 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c47252ccb11fb63bd977e4f12f69260; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58965910, jitterRate=-0.12133947014808655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:12,867 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c47252ccb11fb63bd977e4f12f69260: Writing region info on filesystem at 1733754132850Initializing all the Stores at 1733754132852 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754132852Cleaning up temporary data from old regions at 1733754132858 (+6 ms)Region opened successfully at 1733754132867 (+9 ms) 2024-12-09T14:22:12,867 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c47252ccb11fb63bd977e4f12f69260, disabling compactions & flushes 2024-12-09T14:22:12,867 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260. 2024-12-09T14:22:12,867 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260. 2024-12-09T14:22:12,867 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260. after waiting 0 ms 2024-12-09T14:22:12,867 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260. 2024-12-09T14:22:12,868 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260. 2024-12-09T14:22:12,868 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c47252ccb11fb63bd977e4f12f69260: Waiting for close lock at 1733754132867Disabling compacts and flushes for region at 1733754132867Disabling writes for close at 1733754132867Writing region close event to WAL at 1733754132868 (+1 ms)Closed at 1733754132868 2024-12-09T14:22:12,872 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,872 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,872 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:12,872 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:12,873 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,873 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,873 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:12,873 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:12,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741837_1013 (size=95) 2024-12-09T14:22:12,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741837_1013 (size=95) 2024-12-09T14:22:12,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:12,881 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12852764:(num 1733754132796) 2024-12-09T14:22:12,883 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:22:12,890 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,890 WARN [IPC Server handler 0 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:12,890 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741839_1015 (size=320) 2024-12-09T14:22:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741839_1015 (size=320) 2024-12-09T14:22:12,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:22:12,904 WARN [IPC Server handler 4 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:12,905 WARN [IPC Server handler 4 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:12,905 WARN [IPC Server handler 4 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:12,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741840_1016 (size=253) 2024-12-09T14:22:12,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741840_1016 (size=253) 2024-12-09T14:22:12,939 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1, size=320 (320bytes) 2024-12-09T14:22:12,940 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T14:22:12,940 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T14:22:12,940 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1 2024-12-09T14:22:12,946 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1 after 4ms 2024-12-09T14:22:12,952 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:12,953 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1 took 16ms 2024-12-09T14:22:12,963 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1 so closing down 2024-12-09T14:22:12,963 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:12,966 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T14:22:12,968 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T14:22:12,969 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741841_1017 (size=320) 2024-12-09T14:22:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741841_1017 (size=320) 2024-12-09T14:22:12,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741841_1017 (size=320) 2024-12-09T14:22:12,981 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:12,985 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002 2024-12-09T14:22:12,988 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-09T14:22:12,988 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1, journal: Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1, size=320 (320bytes) at 1733754132939Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1 so closing down at 1733754132963 (+24 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000001-wal-1.temp at 1733754132968 (+5 ms)3 split writer threads finished at 1733754132969 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733754132981 (+12 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002 at 1733754132985 (+4 ms)Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733754132988 (+3 ms) 2024-12-09T14:22:13,004 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2, size=253 (253bytes) 2024-12-09T14:22:13,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2 2024-12-09T14:22:13,005 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2 after 1ms 2024-12-09T14:22:13,009 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:13,009 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2 took 5ms 2024-12-09T14:22:13,012 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2 so closing down 2024-12-09T14:22:13,012 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:13,016 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T14:22:13,018 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T14:22:13,018 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:13,021 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:13,021 WARN [IPC Server handler 0 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:13,021 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:13,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741842_1018 (size=253) 2024-12-09T14:22:13,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741842_1018 (size=253) 2024-12-09T14:22:13,029 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:13,034 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:13,037 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-12-09T14:22:13,039 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-09T14:22:13,039 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2, journal: Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2, size=253 (253bytes) at 1733754133004Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2 so closing down at 1733754133012 (+8 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002-wal-2.temp at 1733754133018 (+6 ms)3 split writer threads finished at 1733754133018Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733754133029 (+11 ms)Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733754133039 (+10 ms) 2024-12-09T14:22:13,039 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:13,043 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:13,065 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal.1733754133044, exclude list is [], retry=0 2024-12-09T14:22:13,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:13,071 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:13,071 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:13,074 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal.1733754133044 2024-12-09T14:22:13,074 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:13,074 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c47252ccb11fb63bd977e4f12f69260, NAME => 'testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:13,075 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:13,075 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,075 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,077 INFO [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,078 INFO [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c47252ccb11fb63bd977e4f12f69260 columnFamilyName a 2024-12-09T14:22:13,078 DEBUG [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:13,079 INFO [StoreOpener-1c47252ccb11fb63bd977e4f12f69260-1 {}] regionserver.HStore(327): Store=1c47252ccb11fb63bd977e4f12f69260/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:13,079 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,080 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,085 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002 2024-12-09T14:22:13,088 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:13,094 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002 2024-12-09T14:22:13,097 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c47252ccb11fb63bd977e4f12f69260 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T14:22:13,148 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/.tmp/a/45327d9327d64faf8e7972ae32911636 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733754132881/Put/seqid=0 2024-12-09T14:22:13,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741844_1020 (size=5170) 2024-12-09T14:22:13,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741844_1020 (size=5170) 2024-12-09T14:22:13,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741844_1020 (size=5170) 2024-12-09T14:22:13,163 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/.tmp/a/45327d9327d64faf8e7972ae32911636 2024-12-09T14:22:13,207 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/.tmp/a/45327d9327d64faf8e7972ae32911636 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/a/45327d9327d64faf8e7972ae32911636 2024-12-09T14:22:13,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/a/45327d9327d64faf8e7972ae32911636, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T14:22:13,223 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1c47252ccb11fb63bd977e4f12f69260 in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T14:22:13,225 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/0000000000000000002 2024-12-09T14:22:13,226 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,226 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c47252ccb11fb63bd977e4f12f69260 2024-12-09T14:22:13,234 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1c47252ccb11fb63bd977e4f12f69260/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T14:22:13,236 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c47252ccb11fb63bd977e4f12f69260; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71463616, jitterRate=0.06489086151123047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:13,236 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c47252ccb11fb63bd977e4f12f69260: Writing region info on filesystem at 1733754133075Initializing all the Stores at 1733754133076 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754133076Obtaining lock to block concurrent updates at 1733754133097 (+21 ms)Preparing flush snapshotting stores in 1c47252ccb11fb63bd977e4f12f69260 at 1733754133097Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733754133100 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733754132786.1c47252ccb11fb63bd977e4f12f69260. at 1733754133100Flushing 1c47252ccb11fb63bd977e4f12f69260/a: creating writer at 1733754133101 (+1 ms)Flushing 1c47252ccb11fb63bd977e4f12f69260/a: appending metadata at 1733754133137 (+36 ms)Flushing 1c47252ccb11fb63bd977e4f12f69260/a: closing flushed file at 1733754133140 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5573dd41: reopening flushed file at 1733754133205 (+65 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1c47252ccb11fb63bd977e4f12f69260 in 124ms, sequenceid=2, compaction requested=false; wal=null at 1733754133223 (+18 ms)Cleaning up temporary data from old regions at 1733754133226 (+3 ms)Region opened successfully at 1733754133236 (+10 ms) 2024-12-09T14:22:13,279 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=369 (was 359) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:53754 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:44364 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:41008 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:44512 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43873/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:41092 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=691 (was 599) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=553 (was 553), ProcessCount=11 (was 11), AvailableMemoryMB=6518 (was 6551) 2024-12-09T14:22:13,293 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=369, OpenFileDescriptor=691, MaxFileDescriptor=1048576, SystemLoadAverage=553, ProcessCount=11, AvailableMemoryMB=6515 2024-12-09T14:22:13,318 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:13,322 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:13,323 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:13,327 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36225227, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-36225227, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:13,346 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36225227/hregion-36225227.1733754133328, exclude list is [], retry=0 2024-12-09T14:22:13,351 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:13,351 WARN [IPC Server handler 2 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:13,351 WARN [IPC Server handler 2 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:13,355 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:13,357 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:13,365 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36225227/hregion-36225227.1733754133328 2024-12-09T14:22:13,366 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:13,366 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1d356da5b6ec1d054be16af7603b66f0, NAME => 'testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:13,371 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:13,371 WARN [IPC Server handler 0 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:13,371 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:13,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741846_1022 (size=64) 2024-12-09T14:22:13,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741846_1022 (size=64) 2024-12-09T14:22:13,782 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:13,784 INFO [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,787 INFO [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d356da5b6ec1d054be16af7603b66f0 columnFamilyName a 2024-12-09T14:22:13,787 DEBUG [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:13,788 INFO [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] regionserver.HStore(327): Store=1d356da5b6ec1d054be16af7603b66f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:13,788 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,791 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,791 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,794 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:13,798 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:13,799 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1d356da5b6ec1d054be16af7603b66f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61771630, jitterRate=-0.0795309841632843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:13,799 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1d356da5b6ec1d054be16af7603b66f0: Writing region info on filesystem at 1733754133782Initializing all the Stores at 1733754133783 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754133783Cleaning up temporary data from old regions at 1733754133791 (+8 ms)Region opened successfully at 1733754133799 (+8 ms) 2024-12-09T14:22:13,800 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1d356da5b6ec1d054be16af7603b66f0, disabling compactions & flushes 2024-12-09T14:22:13,800 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0. 2024-12-09T14:22:13,800 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0. 2024-12-09T14:22:13,800 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0. after waiting 0 ms 2024-12-09T14:22:13,800 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0. 2024-12-09T14:22:13,800 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0. 2024-12-09T14:22:13,800 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1d356da5b6ec1d054be16af7603b66f0: Waiting for close lock at 1733754133799Disabling compacts and flushes for region at 1733754133799Disabling writes for close at 1733754133800 (+1 ms)Writing region close event to WAL at 1733754133800Closed at 1733754133800 2024-12-09T14:22:13,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741845_1021 (size=95) 2024-12-09T14:22:13,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741845_1021 (size=95) 2024-12-09T14:22:13,809 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:13,809 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36225227:(num 1733754133328) 2024-12-09T14:22:13,811 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:22:13,815 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-09T14:22:13,815 WARN [IPC Server handler 0 on default port 43873 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-09T14:22:13,816 WARN [IPC Server handler 0 on default port 43873 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-09T14:22:13,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741847_1023 (size=320) 2024-12-09T14:22:13,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741847_1023 (size=320) 2024-12-09T14:22:13,851 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:22:13,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741848_1024 (size=253) 2024-12-09T14:22:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741848_1024 (size=253) 2024-12-09T14:22:13,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741848_1024 (size=253) 2024-12-09T14:22:13,902 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2, size=253 (253bytes) 2024-12-09T14:22:13,902 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2 2024-12-09T14:22:13,904 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2 after 1ms 2024-12-09T14:22:13,909 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:13,909 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2 took 7ms 2024-12-09T14:22:13,914 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2 so closing down 2024-12-09T14:22:13,914 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:13,917 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T14:22:13,919 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T14:22:13,920 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:13,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741849_1025 (size=253) 2024-12-09T14:22:13,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741849_1025 (size=253) 2024-12-09T14:22:13,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741849_1025 (size=253) 2024-12-09T14:22:13,950 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:13,957 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 2024-12-09T14:22:13,958 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-09T14:22:13,958 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2, journal: Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2, size=253 (253bytes) at 1733754133902Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2 so closing down at 1733754133914 (+12 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002-wal-2.temp at 1733754133919 (+5 ms)3 split writer threads finished at 1733754133920 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733754133950 (+30 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 at 1733754133957 (+7 ms)Processed 1 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733754133958 (+1 ms) 2024-12-09T14:22:13,978 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1, size=320 (320bytes) 2024-12-09T14:22:13,978 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1 2024-12-09T14:22:13,979 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1 after 1ms 2024-12-09T14:22:13,984 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:13,984 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1 took 7ms 2024-12-09T14:22:13,989 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1 so closing down 2024-12-09T14:22:13,989 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:13,991 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T14:22:13,993 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T14:22:13,994 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:14,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741850_1026 (size=320) 2024-12-09T14:22:14,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741850_1026 (size=320) 2024-12-09T14:22:14,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741850_1026 (size=320) 2024-12-09T14:22:14,011 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:14,020 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:14,026 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002, length=253 2024-12-09T14:22:14,029 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 2024-12-09T14:22:14,029 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-09T14:22:14,030 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1, journal: Splitting hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1, size=320 (320bytes) at 1733754133978Finishing writing output for hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1 so closing down at 1733754133989 (+11 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000001-wal-1.temp at 1733754133993 (+4 ms)3 split writer threads finished at 1733754133994 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733754134011 (+17 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 at 1733754134029 (+18 ms)Processed 2 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733754134030 (+1 ms) 2024-12-09T14:22:14,030 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:14,039 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:14,059 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal.1733754134040, exclude list is [], retry=0 2024-12-09T14:22:14,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:14,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:14,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:14,083 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal.1733754134040 2024-12-09T14:22:14,084 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:14,084 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d356da5b6ec1d054be16af7603b66f0, NAME => 'testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:14,084 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:14,084 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,084 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,086 INFO [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,089 INFO [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d356da5b6ec1d054be16af7603b66f0 columnFamilyName a 2024-12-09T14:22:14,089 DEBUG [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:14,090 INFO [StoreOpener-1d356da5b6ec1d054be16af7603b66f0-1 {}] regionserver.HStore(327): Store=1d356da5b6ec1d054be16af7603b66f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:14,090 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,091 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,093 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,094 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 2024-12-09T14:22:14,098 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:14,100 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 2024-12-09T14:22:14,100 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1d356da5b6ec1d054be16af7603b66f0 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T14:22:14,118 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/.tmp/a/d957a31d89d84ec484d07b25cb7e3263 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733754133809/Put/seqid=0 2024-12-09T14:22:14,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741852_1028 (size=5170) 2024-12-09T14:22:14,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741852_1028 (size=5170) 2024-12-09T14:22:14,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741852_1028 (size=5170) 2024-12-09T14:22:14,128 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/.tmp/a/d957a31d89d84ec484d07b25cb7e3263 2024-12-09T14:22:14,136 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/.tmp/a/d957a31d89d84ec484d07b25cb7e3263 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/a/d957a31d89d84ec484d07b25cb7e3263 2024-12-09T14:22:14,143 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/a/d957a31d89d84ec484d07b25cb7e3263, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T14:22:14,144 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1d356da5b6ec1d054be16af7603b66f0 in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T14:22:14,144 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/0000000000000000002 2024-12-09T14:22:14,145 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,145 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,147 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1d356da5b6ec1d054be16af7603b66f0 2024-12-09T14:22:14,150 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/1d356da5b6ec1d054be16af7603b66f0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T14:22:14,151 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1d356da5b6ec1d054be16af7603b66f0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72879346, jitterRate=0.08598688244819641}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:14,151 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1d356da5b6ec1d054be16af7603b66f0: Writing region info on filesystem at 1733754134084Initializing all the Stores at 1733754134086 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754134086Obtaining lock to block concurrent updates at 1733754134100 (+14 ms)Preparing flush snapshotting stores in 1d356da5b6ec1d054be16af7603b66f0 at 1733754134100Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733754134100Flushing stores of testReplayEditsWrittenIntoWAL,,1733754133319.1d356da5b6ec1d054be16af7603b66f0. at 1733754134100Flushing 1d356da5b6ec1d054be16af7603b66f0/a: creating writer at 1733754134100Flushing 1d356da5b6ec1d054be16af7603b66f0/a: appending metadata at 1733754134116 (+16 ms)Flushing 1d356da5b6ec1d054be16af7603b66f0/a: closing flushed file at 1733754134117 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ca2ac0: reopening flushed file at 1733754134135 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1d356da5b6ec1d054be16af7603b66f0 in 43ms, sequenceid=2, compaction requested=false; wal=null at 1733754134144 (+9 ms)Cleaning up temporary data from old regions at 1733754134145 (+1 ms)Region opened successfully at 1733754134151 (+6 ms) 2024-12-09T14:22:14,168 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=378 (was 369) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:44364 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:41008 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:53634 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:41124 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:44586 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:53824 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=773 (was 691) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=553 (was 553), ProcessCount=11 (was 11), AvailableMemoryMB=6451 (was 6515) 2024-12-09T14:22:14,181 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=378, OpenFileDescriptor=773, MaxFileDescriptor=1048576, SystemLoadAverage=553, ProcessCount=11, AvailableMemoryMB=6450 2024-12-09T14:22:14,200 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:14,203 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:14,204 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:14,212 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92918190, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-92918190, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:14,228 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92918190/hregion-92918190.1733754134212, exclude list is [], retry=0 2024-12-09T14:22:14,231 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:14,232 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:14,232 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:14,235 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92918190/hregion-92918190.1733754134212 2024-12-09T14:22:14,235 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:14,235 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4ba8fd46da798bcbf74e7092e0dcb2fa, NAME => 'testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:14,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741854_1030 (size=64) 2024-12-09T14:22:14,249 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741854_1030 (size=64) 2024-12-09T14:22:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741854_1030 (size=64) 2024-12-09T14:22:14,253 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,255 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba8fd46da798bcbf74e7092e0dcb2fa columnFamilyName a 2024-12-09T14:22:14,255 DEBUG [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:14,256 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(327): Store=4ba8fd46da798bcbf74e7092e0dcb2fa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:14,256 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,258 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba8fd46da798bcbf74e7092e0dcb2fa columnFamilyName b 2024-12-09T14:22:14,259 DEBUG [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:14,259 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(327): Store=4ba8fd46da798bcbf74e7092e0dcb2fa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:14,259 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,261 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba8fd46da798bcbf74e7092e0dcb2fa columnFamilyName c 2024-12-09T14:22:14,261 DEBUG [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:14,262 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(327): Store=4ba8fd46da798bcbf74e7092e0dcb2fa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:14,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,265 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,266 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:14,268 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:14,272 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:14,272 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4ba8fd46da798bcbf74e7092e0dcb2fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71331605, jitterRate=0.06292374432086945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:14,273 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4ba8fd46da798bcbf74e7092e0dcb2fa: Writing region info on filesystem at 1733754134249Initializing all the Stores at 1733754134252 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754134252Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754134253 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754134253Cleaning up temporary data from old regions at 1733754134265 (+12 ms)Region opened successfully at 1733754134272 (+7 ms) 2024-12-09T14:22:14,273 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4ba8fd46da798bcbf74e7092e0dcb2fa, disabling compactions & flushes 2024-12-09T14:22:14,273 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:14,273 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:14,273 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. after waiting 0 ms 2024-12-09T14:22:14,273 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:14,274 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:14,274 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4ba8fd46da798bcbf74e7092e0dcb2fa: Waiting for close lock at 1733754134273Disabling compacts and flushes for region at 1733754134273Disabling writes for close at 1733754134273Writing region close event to WAL at 1733754134274 (+1 ms)Closed at 1733754134274 2024-12-09T14:22:14,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741853_1029 (size=95) 2024-12-09T14:22:14,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741853_1029 (size=95) 2024-12-09T14:22:14,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741853_1029 (size=95) 2024-12-09T14:22:14,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:14,280 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92918190:(num 1733754134212) 2024-12-09T14:22:14,280 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:14,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:14,297 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283, exclude list is [], retry=0 2024-12-09T14:22:14,301 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:14,301 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:14,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:14,304 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 2024-12-09T14:22:14,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:14,519 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283, size=0 (0bytes) 2024-12-09T14:22:14,519 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 might be still open, length is 0 2024-12-09T14:22:14,519 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 2024-12-09T14:22:14,521 WARN [IPC Server handler 2 on default port 43873 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T14:22:14,521 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 after 2ms 2024-12-09T14:22:16,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741832_1008 (size=32) 2024-12-09T14:22:16,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741831_1007 (size=1321) 2024-12-09T14:22:17,102 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:53874 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34887:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53874 dst: /127.0.0.1:34887 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34887 remote=/127.0.0.1:53874]. Total timeout mills is 60000, 57379 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:17,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:41166 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35501:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41166 dst: /127.0.0.1:35501 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:17,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:44616 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44616 dst: /127.0.0.1:34769 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:17,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741855_1032 (size=263633) 2024-12-09T14:22:17,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741855_1032 (size=263633) 2024-12-09T14:22:17,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741855_1032 (size=263633) 2024-12-09T14:22:17,865 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:22:17,911 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T14:22:18,523 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 after 4004ms 2024-12-09T14:22:18,528 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:18,532 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 took 4013ms 2024-12-09T14:22:18,567 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733754134283.temp 2024-12-09T14:22:18,579 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000000001-wal.1733754134283.temp 2024-12-09T14:22:18,775 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283; continuing. 2024-12-09T14:22:18,776 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 so closing down 2024-12-09T14:22:18,776 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:18,786 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:18,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741856_1033 (size=263641) 2024-12-09T14:22:18,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741856_1033 (size=263641) 2024-12-09T14:22:18,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741856_1033 (size=263641) 2024-12-09T14:22:18,808 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000000001-wal.1733754134283.temp (wrote 3002 edits, skipped 0 edits in 99 ms) 2024-12-09T14:22:18,819 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000000001-wal.1733754134283.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002 2024-12-09T14:22:18,819 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 283 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:22:18,820 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283, journal: Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283, size=0 (0bytes) at 1733754134519Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000000001-wal.1733754134283.temp at 1733754138579 (+4060 ms)Split 1024 edits, skipped 0 edits. at 1733754138688 (+109 ms)Split 2048 edits, skipped 0 edits. at 1733754138746 (+58 ms)Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 so closing down at 1733754138776 (+30 ms)3 split writer threads finished at 1733754138786 (+10 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000000001-wal.1733754134283.temp (wrote 3002 edits, skipped 0 edits in 99 ms) at 1733754138808 (+22 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000000001-wal.1733754134283.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002 at 1733754138819 (+11 ms)Processed 3002 edits across 1 Regions in 283 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283, size=0, length=0, corrupted=false, cancelled=false at 1733754138820 (+1 ms) 2024-12-09T14:22:18,831 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754134283 2024-12-09T14:22:18,840 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002 2024-12-09T14:22:18,841 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:18,855 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:18,884 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754138856, exclude list is [], retry=0 2024-12-09T14:22:18,892 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:18,892 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:18,893 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:18,927 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754138856 2024-12-09T14:22:18,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:18,932 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:18,940 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:18,947 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba8fd46da798bcbf74e7092e0dcb2fa columnFamilyName a 2024-12-09T14:22:18,947 DEBUG [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:18,949 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(327): Store=4ba8fd46da798bcbf74e7092e0dcb2fa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:18,950 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:18,951 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba8fd46da798bcbf74e7092e0dcb2fa columnFamilyName b 2024-12-09T14:22:18,953 DEBUG [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:18,954 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(327): Store=4ba8fd46da798bcbf74e7092e0dcb2fa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:18,954 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:18,963 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ba8fd46da798bcbf74e7092e0dcb2fa columnFamilyName c 2024-12-09T14:22:18,963 DEBUG [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:18,966 INFO [StoreOpener-4ba8fd46da798bcbf74e7092e0dcb2fa-1 {}] regionserver.HStore(327): Store=4ba8fd46da798bcbf74e7092e0dcb2fa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:18,966 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:18,971 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:18,980 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:18,983 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002 2024-12-09T14:22:18,993 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:19,083 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:22:19,681 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ba8fd46da798bcbf74e7092e0dcb2fa 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T14:22:19,745 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/a/d50d18c9a1784517b79c9a02e7c3c183 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733754134310/Put/seqid=0 2024-12-09T14:22:19,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T14:22:19,766 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:19,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:22:19,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T14:22:19,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T14:22:19,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:19,769 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T14:22:19,770 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:19,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741858_1035 (size=50463) 2024-12-09T14:22:19,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741858_1035 (size=50463) 2024-12-09T14:22:19,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741858_1035 (size=50463) 2024-12-09T14:22:19,803 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/a/d50d18c9a1784517b79c9a02e7c3c183 2024-12-09T14:22:19,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/a/d50d18c9a1784517b79c9a02e7c3c183 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/a/d50d18c9a1784517b79c9a02e7c3c183 2024-12-09T14:22:19,861 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/a/d50d18c9a1784517b79c9a02e7c3c183, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T14:22:19,862 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 4ba8fd46da798bcbf74e7092e0dcb2fa in 180ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T14:22:19,892 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:22:19,892 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ba8fd46da798bcbf74e7092e0dcb2fa 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T14:22:19,921 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/a/bc073e3d83d24afd870ec3d398c4fe60 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733754134342/Put/seqid=0 2024-12-09T14:22:19,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741859_1036 (size=20072) 2024-12-09T14:22:19,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741859_1036 (size=20072) 2024-12-09T14:22:19,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741859_1036 (size=20072) 2024-12-09T14:22:19,979 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/a/bc073e3d83d24afd870ec3d398c4fe60 2024-12-09T14:22:20,022 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/b/49e3d8a46ad041db86a7d3b48fc19333 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733754134370/Put/seqid=0 2024-12-09T14:22:20,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741860_1037 (size=35835) 2024-12-09T14:22:20,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741860_1037 (size=35835) 2024-12-09T14:22:20,073 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/b/49e3d8a46ad041db86a7d3b48fc19333 2024-12-09T14:22:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741860_1037 (size=35835) 2024-12-09T14:22:20,088 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/a/bc073e3d83d24afd870ec3d398c4fe60 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/a/bc073e3d83d24afd870ec3d398c4fe60 2024-12-09T14:22:20,100 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/a/bc073e3d83d24afd870ec3d398c4fe60, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T14:22:20,102 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/b/49e3d8a46ad041db86a7d3b48fc19333 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/b/49e3d8a46ad041db86a7d3b48fc19333 2024-12-09T14:22:20,119 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/b/49e3d8a46ad041db86a7d3b48fc19333, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T14:22:20,119 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 4ba8fd46da798bcbf74e7092e0dcb2fa in 227ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T14:22:20,142 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:22:20,143 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ba8fd46da798bcbf74e7092e0dcb2fa 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T14:22:20,156 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/b/6d3a89f4c2d84edf9a793f128d802cc0 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733754134397/Put/seqid=0 2024-12-09T14:22:20,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741861_1038 (size=35082) 2024-12-09T14:22:20,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741861_1038 (size=35082) 2024-12-09T14:22:20,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741861_1038 (size=35082) 2024-12-09T14:22:20,203 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/b/6d3a89f4c2d84edf9a793f128d802cc0 2024-12-09T14:22:20,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/c/7e5d645be8df45e0b89cb8bc2b214ee0 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733754134436/Put/seqid=0 2024-12-09T14:22:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741862_1039 (size=20825) 2024-12-09T14:22:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741862_1039 (size=20825) 2024-12-09T14:22:20,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741862_1039 (size=20825) 2024-12-09T14:22:20,338 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/c/7e5d645be8df45e0b89cb8bc2b214ee0 2024-12-09T14:22:20,355 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/b/6d3a89f4c2d84edf9a793f128d802cc0 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/b/6d3a89f4c2d84edf9a793f128d802cc0 2024-12-09T14:22:20,375 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/b/6d3a89f4c2d84edf9a793f128d802cc0, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T14:22:20,377 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/c/7e5d645be8df45e0b89cb8bc2b214ee0 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/c/7e5d645be8df45e0b89cb8bc2b214ee0 2024-12-09T14:22:20,389 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/c/7e5d645be8df45e0b89cb8bc2b214ee0, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T14:22:20,389 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 4ba8fd46da798bcbf74e7092e0dcb2fa in 247ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T14:22:20,408 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733754134479/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:20,413 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002 2024-12-09T14:22:20,414 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:22:20,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4ba8fd46da798bcbf74e7092e0dcb2fa 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T14:22:20,429 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/c/12dd825002d94823abb22f50598e690f is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733754134444/Put/seqid=0 2024-12-09T14:22:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741863_1040 (size=50301) 2024-12-09T14:22:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741863_1040 (size=50301) 2024-12-09T14:22:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741863_1040 (size=50301) 2024-12-09T14:22:20,494 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/c/12dd825002d94823abb22f50598e690f 2024-12-09T14:22:20,509 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12dd825002d94823abb22f50598e690f 2024-12-09T14:22:20,514 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/.tmp/c/12dd825002d94823abb22f50598e690f as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/c/12dd825002d94823abb22f50598e690f 2024-12-09T14:22:20,531 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12dd825002d94823abb22f50598e690f 2024-12-09T14:22:20,531 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/c/12dd825002d94823abb22f50598e690f, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T14:22:20,532 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 4ba8fd46da798bcbf74e7092e0dcb2fa in 117ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T14:22:20,538 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/0000000000000003002 2024-12-09T14:22:20,541 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:20,541 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:20,548 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T14:22:20,554 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4ba8fd46da798bcbf74e7092e0dcb2fa 2024-12-09T14:22:20,567 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenIntoWAL/4ba8fd46da798bcbf74e7092e0dcb2fa/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T14:22:20,575 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4ba8fd46da798bcbf74e7092e0dcb2fa; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59181863, jitterRate=-0.11812151968479156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T14:22:20,575 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4ba8fd46da798bcbf74e7092e0dcb2fa: Writing region info on filesystem at 1733754138933Initializing all the Stores at 1733754138939 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754138939Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754138939Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754138939Cleaning up temporary data from old regions at 1733754140541 (+1602 ms)Region opened successfully at 1733754140575 (+34 ms) 2024-12-09T14:22:20,689 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4ba8fd46da798bcbf74e7092e0dcb2fa, disabling compactions & flushes 2024-12-09T14:22:20,690 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:20,690 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:20,690 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. after waiting 0 ms 2024-12-09T14:22:20,690 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:20,743 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733754134201.4ba8fd46da798bcbf74e7092e0dcb2fa. 2024-12-09T14:22:20,743 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4ba8fd46da798bcbf74e7092e0dcb2fa: Waiting for close lock at 1733754140689Disabling compacts and flushes for region at 1733754140689Disabling writes for close at 1733754140690 (+1 ms)Writing region close event to WAL at 1733754140742 (+52 ms)Closed at 1733754140743 (+1 ms) 2024-12-09T14:22:20,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741857_1034 (size=95) 2024-12-09T14:22:20,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741857_1034 (size=95) 2024-12-09T14:22:20,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741857_1034 (size=95) 2024-12-09T14:22:20,767 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:20,767 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733754138856) 2024-12-09T14:22:20,790 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=396 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1341787940_22 at /127.0.0.1:50722 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:43873 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:34121 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1341787940_22 at /127.0.0.1:57756 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:38855 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43873 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34121 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1341787940_22 at /127.0.0.1:37456 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38855 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1341787940_22 at /127.0.0.1:57776 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@60a3f883[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@5bceb9ea[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=853 (was 773) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 553), ProcessCount=11 (was 11), AvailableMemoryMB=6204 (was 6450) 2024-12-09T14:22:20,807 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=396, OpenFileDescriptor=853, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=6203 2024-12-09T14:22:20,832 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:20,836 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:20,837 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:20,846 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72330206, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-72330206, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:20,869 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72330206/hregion-72330206.1733754140846, exclude list is [], retry=0 2024-12-09T14:22:20,880 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:20,883 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:20,887 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:20,892 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72330206/hregion-72330206.1733754140846 2024-12-09T14:22:20,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:20,893 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c45d46db339b718c948f9d7bf9153fa0, NAME => 'test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741865_1042 (size=43) 2024-12-09T14:22:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741865_1042 (size=43) 2024-12-09T14:22:20,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741865_1042 (size=43) 2024-12-09T14:22:20,952 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:20,955 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,958 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c45d46db339b718c948f9d7bf9153fa0 columnFamilyName a 2024-12-09T14:22:20,958 DEBUG [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:20,964 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(327): Store=c45d46db339b718c948f9d7bf9153fa0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:20,965 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,967 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c45d46db339b718c948f9d7bf9153fa0 columnFamilyName b 2024-12-09T14:22:20,967 DEBUG [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:20,968 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(327): Store=c45d46db339b718c948f9d7bf9153fa0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:20,968 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,970 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c45d46db339b718c948f9d7bf9153fa0 columnFamilyName c 2024-12-09T14:22:20,971 DEBUG [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:20,972 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(327): Store=c45d46db339b718c948f9d7bf9153fa0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:20,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,975 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,975 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,976 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:20,979 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:20,982 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:20,986 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c45d46db339b718c948f9d7bf9153fa0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65880741, jitterRate=-0.018300458788871765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:20,988 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c45d46db339b718c948f9d7bf9153fa0: Writing region info on filesystem at 1733754140952Initializing all the Stores at 1733754140954 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754140955 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754140955Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754140955Cleaning up temporary data from old regions at 1733754140975 (+20 ms)Region opened successfully at 1733754140988 (+13 ms) 2024-12-09T14:22:20,988 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c45d46db339b718c948f9d7bf9153fa0, disabling compactions & flushes 2024-12-09T14:22:20,988 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:20,988 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:20,989 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. after waiting 1 ms 2024-12-09T14:22:20,989 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:20,995 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:20,996 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c45d46db339b718c948f9d7bf9153fa0: Waiting for close lock at 1733754140988Disabling compacts and flushes for region at 1733754140988Disabling writes for close at 1733754140989 (+1 ms)Writing region close event to WAL at 1733754140995 (+6 ms)Closed at 1733754140995 2024-12-09T14:22:21,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741864_1041 (size=95) 2024-12-09T14:22:21,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741864_1041 (size=95) 2024-12-09T14:22:21,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741864_1041 (size=95) 2024-12-09T14:22:21,025 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:21,025 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72330206:(num 1733754140846) 2024-12-09T14:22:21,026 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:21,029 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:21,050 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030, exclude list is [], retry=0 2024-12-09T14:22:21,061 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:21,062 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:21,062 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:21,071 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 2024-12-09T14:22:21,072 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:21,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741866_1043 (size=263359) 2024-12-09T14:22:21,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741866_1043 (size=263359) 2024-12-09T14:22:21,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741866_1043 (size=263359) 2024-12-09T14:22:21,391 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030, size=257.2 K (263359bytes) 2024-12-09T14:22:21,391 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 2024-12-09T14:22:21,393 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 after 2ms 2024-12-09T14:22:21,411 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:21,435 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 took 44ms 2024-12-09T14:22:21,470 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733754141030.temp 2024-12-09T14:22:21,482 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000000001-wal.1733754141030.temp 2024-12-09T14:22:21,596 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 so closing down 2024-12-09T14:22:21,597 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:21,603 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741867_1044 (size=263359) 2024-12-09T14:22:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741867_1044 (size=263359) 2024-12-09T14:22:21,620 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000000001-wal.1733754141030.temp (wrote 3000 edits, skipped 0 edits in 71 ms) 2024-12-09T14:22:21,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741867_1044 (size=263359) 2024-12-09T14:22:21,630 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000000001-wal.1733754141030.temp to hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000 2024-12-09T14:22:21,631 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 174 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-12-09T14:22:21,631 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030, journal: Splitting hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030, size=257.2 K (263359bytes) at 1733754141391Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000000001-wal.1733754141030.temp at 1733754141483 (+92 ms)Split 1024 edits, skipped 0 edits. at 1733754141504 (+21 ms)Split 2048 edits, skipped 0 edits. at 1733754141566 (+62 ms)Finishing writing output for hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 so closing down at 1733754141596 (+30 ms)3 split writer threads finished at 1733754141603 (+7 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000000001-wal.1733754141030.temp (wrote 3000 edits, skipped 0 edits in 71 ms) at 1733754141620 (+17 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000000001-wal.1733754141030.temp to hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000 at 1733754141630 (+10 ms)Processed 3000 edits across 1 Regions in 174 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1733754141631 (+1 ms) 2024-12-09T14:22:21,635 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141030 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754141030 2024-12-09T14:22:21,636 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000 2024-12-09T14:22:21,637 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:21,643 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:21,665 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643, exclude list is [], retry=0 2024-12-09T14:22:21,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:21,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:21,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:21,684 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 2024-12-09T14:22:21,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:21,935 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 not finished, retry = 0 2024-12-09T14:22:21,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741868_1045 (size=263486) 2024-12-09T14:22:21,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741868_1045 (size=263486) 2024-12-09T14:22:21,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741868_1045 (size=263486) 2024-12-09T14:22:22,066 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643, size=257.3 K (263486bytes) 2024-12-09T14:22:22,066 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 2024-12-09T14:22:22,067 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 after 0ms 2024-12-09T14:22:22,071 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:22,073 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 took 6ms 2024-12-09T14:22:22,084 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733754141643.temp 2024-12-09T14:22:22,090 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003001-wal.1733754141643.temp 2024-12-09T14:22:22,177 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 so closing down 2024-12-09T14:22:22,177 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:22,178 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741869_1046 (size=263486) 2024-12-09T14:22:22,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741869_1046 (size=263486) 2024-12-09T14:22:22,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741869_1046 (size=263486) 2024-12-09T14:22:22,588 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003001-wal.1733754141643.temp (wrote 3000 edits, skipped 0 edits in 55 ms) 2024-12-09T14:22:22,590 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003001-wal.1733754141643.temp to hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000 2024-12-09T14:22:22,590 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 512 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-12-09T14:22:22,591 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643, journal: Splitting hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643, size=257.3 K (263486bytes) at 1733754142066Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003001-wal.1733754141643.temp at 1733754142091 (+25 ms)Split 1024 edits, skipped 0 edits. at 1733754142103 (+12 ms)Split 2048 edits, skipped 0 edits. at 1733754142139 (+36 ms)Finishing writing output for hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 so closing down at 1733754142177 (+38 ms)3 split writer threads finished at 1733754142178 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003001-wal.1733754141643.temp (wrote 3000 edits, skipped 0 edits in 55 ms) at 1733754142588 (+410 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003001-wal.1733754141643.temp to hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000 at 1733754142590 (+2 ms)Processed 3000 edits across 1 Regions in 512 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1733754142590 2024-12-09T14:22:22,593 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754141643 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754141643 2024-12-09T14:22:22,595 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000 2024-12-09T14:22:22,595 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:22,597 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/test2727-manual,16010,1733754140831, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:22,623 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754142598, exclude list is [], retry=0 2024-12-09T14:22:22,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:22,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:22,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:22,651 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754142598 2024-12-09T14:22:22,652 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:22,652 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c45d46db339b718c948f9d7bf9153fa0, NAME => 'test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:22,652 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:22,652 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,652 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,657 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,662 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c45d46db339b718c948f9d7bf9153fa0 columnFamilyName a 2024-12-09T14:22:22,663 DEBUG [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:22,664 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(327): Store=c45d46db339b718c948f9d7bf9153fa0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:22,664 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,666 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c45d46db339b718c948f9d7bf9153fa0 columnFamilyName b 2024-12-09T14:22:22,667 DEBUG [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:22,668 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(327): Store=c45d46db339b718c948f9d7bf9153fa0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:22,669 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,670 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c45d46db339b718c948f9d7bf9153fa0 columnFamilyName c 2024-12-09T14:22:22,670 DEBUG [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:22,671 INFO [StoreOpener-c45d46db339b718c948f9d7bf9153fa0-1 {}] regionserver.HStore(327): Store=c45d46db339b718c948f9d7bf9153fa0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:22,671 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,677 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:22,679 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000 2024-12-09T14:22:22,683 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:22,754 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000 2024-12-09T14:22:22,759 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000 2024-12-09T14:22:22,764 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:22,837 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000 2024-12-09T14:22:22,837 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c45d46db339b718c948f9d7bf9153fa0 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T14:22:22,880 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/a/c233a9d120e845b286c985f008622351 is 41, key is test2727/a:100/1733754141689/Put/seqid=0 2024-12-09T14:22:22,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741871_1048 (size=84227) 2024-12-09T14:22:22,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741871_1048 (size=84227) 2024-12-09T14:22:22,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741871_1048 (size=84227) 2024-12-09T14:22:22,959 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/a/c233a9d120e845b286c985f008622351 2024-12-09T14:22:23,014 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/b/fbe911da07264c13a7b160463ef34c3a is 41, key is test2727/b:100/1733754141753/Put/seqid=0 2024-12-09T14:22:23,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741872_1049 (size=84609) 2024-12-09T14:22:23,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741872_1049 (size=84609) 2024-12-09T14:22:23,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741872_1049 (size=84609) 2024-12-09T14:22:23,075 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/b/fbe911da07264c13a7b160463ef34c3a 2024-12-09T14:22:23,141 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/c/97b379c1edca4a248ee6c24e45ed9672 is 41, key is test2727/c:100/1733754141825/Put/seqid=0 2024-12-09T14:22:23,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741873_1050 (size=84609) 2024-12-09T14:22:23,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741873_1050 (size=84609) 2024-12-09T14:22:23,227 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/c/97b379c1edca4a248ee6c24e45ed9672 2024-12-09T14:22:23,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741873_1050 (size=84609) 2024-12-09T14:22:23,264 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/a/c233a9d120e845b286c985f008622351 as hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/a/c233a9d120e845b286c985f008622351 2024-12-09T14:22:23,286 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/a/c233a9d120e845b286c985f008622351, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T14:22:23,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/b/fbe911da07264c13a7b160463ef34c3a as hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/b/fbe911da07264c13a7b160463ef34c3a 2024-12-09T14:22:23,311 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/b/fbe911da07264c13a7b160463ef34c3a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T14:22:23,314 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/.tmp/c/97b379c1edca4a248ee6c24e45ed9672 as hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/c/97b379c1edca4a248ee6c24e45ed9672 2024-12-09T14:22:23,333 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/c/97b379c1edca4a248ee6c24e45ed9672, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T14:22:23,334 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for c45d46db339b718c948f9d7bf9153fa0 in 496ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T14:22:23,338 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000003000 2024-12-09T14:22:23,341 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/0000000000000006000 2024-12-09T14:22:23,343 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:23,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:23,345 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:23,353 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c45d46db339b718c948f9d7bf9153fa0 2024-12-09T14:22:23,362 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/test2727/c45d46db339b718c948f9d7bf9153fa0/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T14:22:23,364 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c45d46db339b718c948f9d7bf9153fa0; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59199514, jitterRate=-0.11785849928855896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:23,366 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c45d46db339b718c948f9d7bf9153fa0: Writing region info on filesystem at 1733754142653Initializing all the Stores at 1733754142656 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754142656Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754142656Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754142656Obtaining lock to block concurrent updates at 1733754142837 (+181 ms)Preparing flush snapshotting stores in c45d46db339b718c948f9d7bf9153fa0 at 1733754142837Finished memstore snapshotting test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733754142838 (+1 ms)Flushing stores of test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. at 1733754142838Flushing c45d46db339b718c948f9d7bf9153fa0/a: creating writer at 1733754142838Flushing c45d46db339b718c948f9d7bf9153fa0/a: appending metadata at 1733754142879 (+41 ms)Flushing c45d46db339b718c948f9d7bf9153fa0/a: closing flushed file at 1733754142879Flushing c45d46db339b718c948f9d7bf9153fa0/b: creating writer at 1733754142973 (+94 ms)Flushing c45d46db339b718c948f9d7bf9153fa0/b: appending metadata at 1733754143012 (+39 ms)Flushing c45d46db339b718c948f9d7bf9153fa0/b: closing flushed file at 1733754143012Flushing c45d46db339b718c948f9d7bf9153fa0/c: creating writer at 1733754143093 (+81 ms)Flushing c45d46db339b718c948f9d7bf9153fa0/c: appending metadata at 1733754143138 (+45 ms)Flushing c45d46db339b718c948f9d7bf9153fa0/c: closing flushed file at 1733754143139 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101d42aa: reopening flushed file at 1733754143256 (+117 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19d080a5: reopening flushed file at 1733754143286 (+30 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f64df75: reopening flushed file at 1733754143311 (+25 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for c45d46db339b718c948f9d7bf9153fa0 in 496ms, sequenceid=6000, compaction requested=false; wal=null at 1733754143334 (+23 ms)Cleaning up temporary data from old regions at 1733754143344 (+10 ms)Region opened successfully at 1733754143366 (+22 ms) 2024-12-09T14:22:23,368 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T14:22:23,368 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c45d46db339b718c948f9d7bf9153fa0, disabling compactions & flushes 2024-12-09T14:22:23,368 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:23,368 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:23,368 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. after waiting 0 ms 2024-12-09T14:22:23,369 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:23,403 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733754140833.c45d46db339b718c948f9d7bf9153fa0. 2024-12-09T14:22:23,404 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c45d46db339b718c948f9d7bf9153fa0: Waiting for close lock at 1733754143368Disabling compacts and flushes for region at 1733754143368Disabling writes for close at 1733754143368Writing region close event to WAL at 1733754143403 (+35 ms)Closed at 1733754143403 2024-12-09T14:22:23,428 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733754140831/wal.1733754142598 not finished, retry = 0 2024-12-09T14:22:23,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741870_1047 (size=95) 2024-12-09T14:22:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741870_1047 (size=95) 2024-12-09T14:22:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741870_1047 (size=95) 2024-12-09T14:22:23,533 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:23,533 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733754142598) 2024-12-09T14:22:23,558 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=395 (was 396), OpenFileDescriptor=909 (was 853) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 524), ProcessCount=11 (was 11), AvailableMemoryMB=5757 (was 6203) 2024-12-09T14:22:23,578 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=395, OpenFileDescriptor=909, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=5757 2024-12-09T14:22:23,697 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:23,717 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:23,718 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733754143717 2024-12-09T14:22:23,788 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717 2024-12-09T14:22:23,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:23,804 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ce31fe5476f68751a08be53086fbc74, NAME => 'testSequentialEditLogSeqNum,,1733754143698.5ce31fe5476f68751a08be53086fbc74.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:23,804 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733754143698.5ce31fe5476f68751a08be53086fbc74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:23,807 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,807 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,809 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74 doesn't exist for region: 5ce31fe5476f68751a08be53086fbc74 on table testSequentialEditLogSeqNum 2024-12-09T14:22:23,814 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 5ce31fe5476f68751a08be53086fbc74 on table testSequentialEditLogSeqNum 2024-12-09T14:22:23,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741875_1052 (size=62) 2024-12-09T14:22:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741875_1052 (size=62) 2024-12-09T14:22:23,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741875_1052 (size=62) 2024-12-09T14:22:23,879 INFO [StoreOpener-5ce31fe5476f68751a08be53086fbc74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,889 INFO [StoreOpener-5ce31fe5476f68751a08be53086fbc74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ce31fe5476f68751a08be53086fbc74 columnFamilyName a 2024-12-09T14:22:23,889 DEBUG [StoreOpener-5ce31fe5476f68751a08be53086fbc74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:23,890 INFO [StoreOpener-5ce31fe5476f68751a08be53086fbc74-1 {}] regionserver.HStore(327): Store=5ce31fe5476f68751a08be53086fbc74/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:23,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,893 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,897 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5ce31fe5476f68751a08be53086fbc74 2024-12-09T14:22:23,903 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:23,904 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5ce31fe5476f68751a08be53086fbc74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73050282, jitterRate=0.08853402733802795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:23,905 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5ce31fe5476f68751a08be53086fbc74: Writing region info on filesystem at 1733754143807Initializing all the Stores at 1733754143872 (+65 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754143872Cleaning up temporary data from old regions at 1733754143893 (+21 ms)Region opened successfully at 1733754143905 (+12 ms) 2024-12-09T14:22:23,954 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5ce31fe5476f68751a08be53086fbc74 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T14:22:23,997 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/.tmp/a/414768f7831a4df59baf09d362d8767e is 81, key is testSequentialEditLogSeqNum/a:x0/1733754143905/Put/seqid=0 2024-12-09T14:22:24,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741876_1053 (size=5833) 2024-12-09T14:22:24,044 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/.tmp/a/414768f7831a4df59baf09d362d8767e 2024-12-09T14:22:24,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741876_1053 (size=5833) 2024-12-09T14:22:24,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741876_1053 (size=5833) 2024-12-09T14:22:24,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/.tmp/a/414768f7831a4df59baf09d362d8767e as hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/a/414768f7831a4df59baf09d362d8767e 2024-12-09T14:22:24,071 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/a/414768f7831a4df59baf09d362d8767e, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T14:22:24,077 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5ce31fe5476f68751a08be53086fbc74 in 122ms, sequenceid=13, compaction requested=false 2024-12-09T14:22:24,077 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5ce31fe5476f68751a08be53086fbc74: 2024-12-09T14:22:24,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:22:24,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:22:24,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:22:24,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:22:24,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:22:24,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741874_1051 (size=1845) 2024-12-09T14:22:24,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741874_1051 (size=1845) 2024-12-09T14:22:24,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741874_1051 (size=1845) 2024-12-09T14:22:24,145 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717, size=1.8 K (1845bytes) 2024-12-09T14:22:24,145 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717 2024-12-09T14:22:24,150 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717 after 5ms 2024-12-09T14:22:24,162 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:24,166 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717 took 22ms 2024-12-09T14:22:24,173 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717 so closing down 2024-12-09T14:22:24,174 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:24,175 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733754143717.temp 2024-12-09T14:22:24,178 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000003-wal.1733754143717.temp 2024-12-09T14:22:24,179 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:24,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741877_1054 (size=1477) 2024-12-09T14:22:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741877_1054 (size=1477) 2024-12-09T14:22:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741877_1054 (size=1477) 2024-12-09T14:22:24,219 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000003-wal.1733754143717.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:24,222 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000003-wal.1733754143717.temp to hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000020 2024-12-09T14:22:24,222 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717, size=1.8 K, length=1845, corrupted=false, cancelled=false 2024-12-09T14:22:24,222 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717, journal: Splitting hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717, size=1.8 K (1845bytes) at 1733754144145Finishing writing output for hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717 so closing down at 1733754144174 (+29 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000003-wal.1733754143717.temp at 1733754144179 (+5 ms)3 split writer threads finished at 1733754144179Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000003-wal.1733754143717.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733754144219 (+40 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000003-wal.1733754143717.temp to hdfs://localhost:43873/hbase/data/default/testSequentialEditLogSeqNum/5ce31fe5476f68751a08be53086fbc74/recovered.edits/0000000000000000020 at 1733754144222 (+3 ms)Processed 17 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:43873/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733754143693/wal.1733754143717, size=1.8 K, length=1845, corrupted=false, cancelled=false at 1733754144222 2024-12-09T14:22:24,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=400 (was 395) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:37552 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:50854 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:57884 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=945 (was 909) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 524), ProcessCount=11 (was 11), AvailableMemoryMB=5706 (was 5757) 2024-12-09T14:22:24,264 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=400, OpenFileDescriptor=945, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=5706 2024-12-09T14:22:24,288 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:24,291 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:24,293 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:24,296 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-88845728, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-88845728, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:24,316 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-88845728/hregion-88845728.1733754144297, exclude list is [], retry=0 2024-12-09T14:22:24,321 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:24,321 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:24,323 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:24,367 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-88845728/hregion-88845728.1733754144297 2024-12-09T14:22:24,368 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675)] 2024-12-09T14:22:24,368 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1a5685506e65b512d603659673f6cd89, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741879_1056 (size=70) 2024-12-09T14:22:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741879_1056 (size=70) 2024-12-09T14:22:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741879_1056 (size=70) 2024-12-09T14:22:24,384 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:24,386 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,388 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName a 2024-12-09T14:22:24,388 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:24,389 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:24,389 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,390 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName b 2024-12-09T14:22:24,390 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:24,391 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:24,391 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,392 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName c 2024-12-09T14:22:24,392 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:24,392 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:24,392 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,395 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,395 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,396 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:24,397 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,399 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:24,400 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a5685506e65b512d603659673f6cd89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71483862, jitterRate=0.06519255042076111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:24,401 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a5685506e65b512d603659673f6cd89: Writing region info on filesystem at 1733754144385Initializing all the Stores at 1733754144386 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754144386Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754144386Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754144386Cleaning up temporary data from old regions at 1733754144395 (+9 ms)Region opened successfully at 1733754144401 (+6 ms) 2024-12-09T14:22:24,402 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1a5685506e65b512d603659673f6cd89, disabling compactions & flushes 2024-12-09T14:22:24,402 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:24,402 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:24,402 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. after waiting 0 ms 2024-12-09T14:22:24,402 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:24,402 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:24,402 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1a5685506e65b512d603659673f6cd89: Waiting for close lock at 1733754144401Disabling compacts and flushes for region at 1733754144401Disabling writes for close at 1733754144402 (+1 ms)Writing region close event to WAL at 1733754144402Closed at 1733754144402 2024-12-09T14:22:24,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741878_1055 (size=95) 2024-12-09T14:22:24,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741878_1055 (size=95) 2024-12-09T14:22:24,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741878_1055 (size=95) 2024-12-09T14:22:24,411 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:24,411 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-88845728:(num 1733754144297) 2024-12-09T14:22:24,411 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:24,413 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:24,428 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413, exclude list is [], retry=0 2024-12-09T14:22:24,431 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:24,431 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:24,432 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:24,433 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 2024-12-09T14:22:24,434 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:24,434 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a5685506e65b512d603659673f6cd89, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:24,434 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:24,434 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,434 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,437 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,438 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName a 2024-12-09T14:22:24,438 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:24,438 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:24,439 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,439 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName b 2024-12-09T14:22:24,439 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:24,440 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:24,440 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,441 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName c 2024-12-09T14:22:24,441 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:24,441 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:24,441 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,442 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,443 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,444 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,444 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,445 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:24,446 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:24,447 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a5685506e65b512d603659673f6cd89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62830321, jitterRate=-0.06375525891780853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:24,448 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a5685506e65b512d603659673f6cd89: Writing region info on filesystem at 1733754144434Initializing all the Stores at 1733754144435 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754144435Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754144437 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754144437Cleaning up temporary data from old regions at 1733754144444 (+7 ms)Region opened successfully at 1733754144448 (+4 ms) 2024-12-09T14:22:24,453 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733754144452/Put/seqid=0 2024-12-09T14:22:24,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741881_1058 (size=4826) 2024-12-09T14:22:24,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741881_1058 (size=4826) 2024-12-09T14:22:24,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741881_1058 (size=4826) 2024-12-09T14:22:24,465 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43873/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 1a5685506e65b512d603659673f6cd89/a 2024-12-09T14:22:24,475 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T14:22:24,475 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:22:24,475 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1a5685506e65b512d603659673f6cd89: 2024-12-09T14:22:24,477 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/0dbdc0f742f84484b6811adf0545d45f_SeqId_3_ 2024-12-09T14:22:24,478 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43873/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1a5685506e65b512d603659673f6cd89/a as hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/0dbdc0f742f84484b6811adf0545d45f_SeqId_3_ - updating store file list. 2024-12-09T14:22:24,484 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0dbdc0f742f84484b6811adf0545d45f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:24,485 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/0dbdc0f742f84484b6811adf0545d45f_SeqId_3_ into 1a5685506e65b512d603659673f6cd89/a 2024-12-09T14:22:24,485 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43873/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1a5685506e65b512d603659673f6cd89/a (new location: hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/0dbdc0f742f84484b6811adf0545d45f_SeqId_3_) 2024-12-09T14:22:24,530 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413, size=0 (0bytes) 2024-12-09T14:22:24,530 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 might be still open, length is 0 2024-12-09T14:22:24,530 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 2024-12-09T14:22:24,530 WARN [IPC Server handler 2 on default port 43873 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T14:22:24,531 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 after 1ms 2024-12-09T14:22:26,080 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:37672 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37672 dst: /127.0.0.1:34769 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34769 remote=/127.0.0.1:37672]. Total timeout mills is 60000, 58412 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:26,081 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:57962 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34887:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57962 dst: /127.0.0.1:34887 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:26,081 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:50942 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35501:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50942 dst: /127.0.0.1:35501 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:26,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741880_1059 (size=473) 2024-12-09T14:22:26,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741880_1059 (size=473) 2024-12-09T14:22:26,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741880_1059 (size=473) 2024-12-09T14:22:28,532 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 after 4002ms 2024-12-09T14:22:28,536 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:28,536 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 took 4007ms 2024-12-09T14:22:28,539 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413; continuing. 2024-12-09T14:22:28,539 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 so closing down 2024-12-09T14:22:28,539 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:28,547 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733754144413.temp 2024-12-09T14:22:28,549 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005-wal.1733754144413.temp 2024-12-09T14:22:28,550 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:28,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741882_1060 (size=259) 2024-12-09T14:22:28,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741882_1060 (size=259) 2024-12-09T14:22:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741882_1060 (size=259) 2024-12-09T14:22:28,561 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005-wal.1733754144413.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:28,563 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005-wal.1733754144413.temp to hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005 2024-12-09T14:22:28,563 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=1; WAL=hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:22:28,563 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413, journal: Splitting hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413, size=0 (0bytes) at 1733754144530Finishing writing output for hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 so closing down at 1733754148539 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005-wal.1733754144413.temp at 1733754148549 (+10 ms)3 split writer threads finished at 1733754148550 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005-wal.1733754144413.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733754148561 (+11 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005-wal.1733754144413.temp to hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005 at 1733754148563 (+2 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=1; WAL=hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413, size=0, length=0, corrupted=false, cancelled=false at 1733754148563 2024-12-09T14:22:28,565 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754144413 2024-12-09T14:22:28,566 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005 2024-12-09T14:22:28,567 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:28,569 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:28,589 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754148570, exclude list is [], retry=0 2024-12-09T14:22:28,592 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:28,593 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:28,593 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:28,596 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754148570 2024-12-09T14:22:28,596 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:28,596 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a5685506e65b512d603659673f6cd89, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:28,597 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:28,597 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,597 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,599 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,600 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName a 2024-12-09T14:22:28,600 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:28,614 DEBUG [StoreFileOpener-1a5685506e65b512d603659673f6cd89-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0dbdc0f742f84484b6811adf0545d45f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:28,615 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/0dbdc0f742f84484b6811adf0545d45f_SeqId_3_ 2024-12-09T14:22:28,615 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:28,615 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,616 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName b 2024-12-09T14:22:28,617 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:28,617 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:28,617 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,618 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a5685506e65b512d603659673f6cd89 columnFamilyName c 2024-12-09T14:22:28,618 DEBUG [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:28,619 INFO [StoreOpener-1a5685506e65b512d603659673f6cd89-1 {}] regionserver.HStore(327): Store=1a5685506e65b512d603659673f6cd89/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:28,619 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,620 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,622 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,623 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005 2024-12-09T14:22:28,627 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:28,628 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005 2024-12-09T14:22:28,628 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a5685506e65b512d603659673f6cd89 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T14:22:28,650 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/.tmp/a/7a01893244c345649b7892ebae1042d4 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733754144491/Put/seqid=0 2024-12-09T14:22:28,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741884_1062 (size=5149) 2024-12-09T14:22:28,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741884_1062 (size=5149) 2024-12-09T14:22:28,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741884_1062 (size=5149) 2024-12-09T14:22:28,665 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/.tmp/a/7a01893244c345649b7892ebae1042d4 2024-12-09T14:22:28,674 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/.tmp/a/7a01893244c345649b7892ebae1042d4 as hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/7a01893244c345649b7892ebae1042d4 2024-12-09T14:22:28,682 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/a/7a01893244c345649b7892ebae1042d4, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T14:22:28,682 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1a5685506e65b512d603659673f6cd89 in 54ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T14:22:28,683 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/0000000000000000005 2024-12-09T14:22:28,685 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,685 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,686 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:28,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a5685506e65b512d603659673f6cd89 2024-12-09T14:22:28,691 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1a5685506e65b512d603659673f6cd89/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T14:22:28,693 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a5685506e65b512d603659673f6cd89; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63957094, jitterRate=-0.04696503281593323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:28,693 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a5685506e65b512d603659673f6cd89: Writing region info on filesystem at 1733754148597Initializing all the Stores at 1733754148599 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754148599Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754148599Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754148599Obtaining lock to block concurrent updates at 1733754148628 (+29 ms)Preparing flush snapshotting stores in 1a5685506e65b512d603659673f6cd89 at 1733754148628Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733754148628Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. at 1733754148628Flushing 1a5685506e65b512d603659673f6cd89/a: creating writer at 1733754148628Flushing 1a5685506e65b512d603659673f6cd89/a: appending metadata at 1733754148649 (+21 ms)Flushing 1a5685506e65b512d603659673f6cd89/a: closing flushed file at 1733754148649Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a0517aa: reopening flushed file at 1733754148673 (+24 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1a5685506e65b512d603659673f6cd89 in 54ms, sequenceid=5, compaction requested=false; wal=null at 1733754148682 (+9 ms)Cleaning up temporary data from old regions at 1733754148685 (+3 ms)Region opened successfully at 1733754148693 (+8 ms) 2024-12-09T14:22:28,698 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1a5685506e65b512d603659673f6cd89, disabling compactions & flushes 2024-12-09T14:22:28,698 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:28,698 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:28,698 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. after waiting 0 ms 2024-12-09T14:22:28,698 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:28,699 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733754144290.1a5685506e65b512d603659673f6cd89. 2024-12-09T14:22:28,699 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1a5685506e65b512d603659673f6cd89: Waiting for close lock at 1733754148698Disabling compacts and flushes for region at 1733754148698Disabling writes for close at 1733754148698Writing region close event to WAL at 1733754148699 (+1 ms)Closed at 1733754148699 2024-12-09T14:22:28,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741883_1061 (size=95) 2024-12-09T14:22:28,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741883_1061 (size=95) 2024-12-09T14:22:28,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741883_1061 (size=95) 2024-12-09T14:22:28,706 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:28,706 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733754148570) 2024-12-09T14:22:28,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=407 (was 400) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:43873 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1308557522_22 at /127.0.0.1:52934 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1308557522_22 at /127.0.0.1:47098 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43873 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1308557522_22 at /127.0.0.1:40158 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1011 (was 945) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=506 (was 524), ProcessCount=11 (was 11), AvailableMemoryMB=5701 (was 5706) 2024-12-09T14:22:28,740 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=407, OpenFileDescriptor=1011, MaxFileDescriptor=1048576, SystemLoadAverage=506, ProcessCount=11, AvailableMemoryMB=5700 2024-12-09T14:22:28,764 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:28,771 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T14:22:28,776 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:28,780 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@dc0fe47 2024-12-09T14:22:28,781 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T14:22:28,784 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T14:22:28,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:22:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T14:22:28,801 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T14:22:28,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T14:22:28,805 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:28,807 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T14:22:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:22:28,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741885_1063 (size=694) 2024-12-09T14:22:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741885_1063 (size=694) 2024-12-09T14:22:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741885_1063 (size=694) 2024-12-09T14:22:28,822 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9 2024-12-09T14:22:28,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741886_1064 (size=77) 2024-12-09T14:22:28,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741886_1064 (size=77) 2024-12-09T14:22:28,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741886_1064 (size=77) 2024-12-09T14:22:28,838 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:28,838 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:28,838 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:28,838 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:28,838 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:28,838 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:28,838 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:28,838 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754148838Disabling compacts and flushes for region at 1733754148838Disabling writes for close at 1733754148838Writing region close event to WAL at 1733754148838Closed at 1733754148838 2024-12-09T14:22:28,841 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T14:22:28,848 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733754148841"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733754148841"}]},"ts":"1733754148841"} 2024-12-09T14:22:28,853 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T14:22:28,855 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T14:22:28,859 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733754148856"}]},"ts":"1733754148856"} 2024-12-09T14:22:28,865 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T14:22:28,866 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {f4e784dc7cb5=0} racks are {/default-rack=0} 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T14:22:28,868 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:22:28,868 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:22:28,868 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T14:22:28,868 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:22:28,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN}] 2024-12-09T14:22:28,874 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN 2024-12-09T14:22:28,875 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN; state=OFFLINE, location=f4e784dc7cb5,43199,1733754130338; forceNewPlan=false, retain=false 2024-12-09T14:22:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:22:29,029 INFO [f4e784dc7cb5:39231 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:22:29,030 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPENING, regionLocation=f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:29,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN because future has completed 2024-12-09T14:22:29,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338}] 2024-12-09T14:22:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:22:29,198 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:22:29,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58571, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:22:29,220 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,220 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:29,221 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,221 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:29,221 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,221 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,227 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,231 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf1 2024-12-09T14:22:29,231 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:29,232 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:29,232 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,234 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf2 2024-12-09T14:22:29,234 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:29,235 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:29,235 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,236 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,237 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,238 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,238 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,241 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:22:29,242 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,266 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:29,268 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 04cac2e3195e2ed23ca8b3c75b748c09; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71015703, jitterRate=0.058216437697410583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:22:29,268 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,269 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 04cac2e3195e2ed23ca8b3c75b748c09: Running coprocessor pre-open hook at 1733754149221Writing region info on filesystem at 1733754149221Initializing all the Stores at 1733754149222 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754149222Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754149226 (+4 ms)Cleaning up temporary data from old regions at 1733754149238 (+12 ms)Running coprocessor post-open hooks at 1733754149268 (+30 ms)Region opened successfully at 1733754149269 (+1 ms) 2024-12-09T14:22:29,274 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., pid=6, masterSystemTime=1733754149197 2024-12-09T14:22:29,279 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,279 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,281 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPEN, openSeqNum=2, regionLocation=f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:29,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338 because future has completed 2024-12-09T14:22:29,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T14:22:29,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338 in 247 msec 2024-12-09T14:22:29,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T14:22:29,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN in 425 msec 2024-12-09T14:22:29,303 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T14:22:29,304 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733754149303"}]},"ts":"1733754149303"} 2024-12-09T14:22:29,308 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T14:22:29,310 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T14:22:29,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 519 msec 2024-12-09T14:22:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:22:29,439 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T14:22:29,439 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T14:22:29,441 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:22:29,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T14:22:29,448 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:22:29,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T14:22:29,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=2] 2024-12-09T14:22:29,468 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:22:29,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:22:29,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=04cac2e3195e2ed23ca8b3c75b748c09, source=f4e784dc7cb5,43199,1733754130338, destination=f4e784dc7cb5,33819,1733754130225, warming up region on f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:22:29,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=04cac2e3195e2ed23ca8b3c75b748c09, source=f4e784dc7cb5,43199,1733754130338, destination=f4e784dc7cb5,33819,1733754130225, running balancer 2024-12-09T14:22:29,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34859, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:22:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE 2024-12-09T14:22:29,496 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE 2024-12-09T14:22:29,499 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=CLOSING, regionLocation=f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:29,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(7855): Warmup {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:29,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE because future has completed 2024-12-09T14:22:29,515 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T14:22:29,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338}] 2024-12-09T14:22:29,516 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,520 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf1 2024-12-09T14:22:29,520 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:29,521 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:29,521 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,522 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf2 2024-12-09T14:22:29,523 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:29,523 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:29,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754149524Disabling compacts and flushes for region at 1733754149524Disabling writes for close at 1733754149524Writing region close event to WAL at 1733754149538 (+14 ms)Closed at 1733754149538 2024-12-09T14:22:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T14:22:29,683 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,683 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T14:22:29,684 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:29,684 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,684 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,684 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:29,684 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,684 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 04cac2e3195e2ed23ca8b3c75b748c09 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T14:22:29,728 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/ef6d12e1fc3749aca8201bca58b6cffd is 35, key is r1/cf1:q/1733754149472/Put/seqid=0 2024-12-09T14:22:29,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T14:22:29,765 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:29,766 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T14:22:29,766 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:29,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T14:22:29,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:29,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741887_1065 (size=4783) 2024-12-09T14:22:29,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741887_1065 (size=4783) 2024-12-09T14:22:29,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741887_1065 (size=4783) 2024-12-09T14:22:29,789 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/ef6d12e1fc3749aca8201bca58b6cffd 2024-12-09T14:22:29,800 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/ef6d12e1fc3749aca8201bca58b6cffd as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd 2024-12-09T14:22:29,810 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T14:22:29,813 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 04cac2e3195e2ed23ca8b3c75b748c09 in 129ms, sequenceid=5, compaction requested=false 2024-12-09T14:22:29,813 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T14:22:29,822 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T14:22:29,825 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:29,825 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754149684Running coprocessor pre-close hooks at 1733754149684Disabling compacts and flushes for region at 1733754149684Disabling writes for close at 1733754149684Obtaining lock to block concurrent updates at 1733754149684Preparing flush snapshotting stores in 04cac2e3195e2ed23ca8b3c75b748c09 at 1733754149684Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733754149685 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. at 1733754149686 (+1 ms)Flushing 04cac2e3195e2ed23ca8b3c75b748c09/cf1: creating writer at 1733754149686Flushing 04cac2e3195e2ed23ca8b3c75b748c09/cf1: appending metadata at 1733754149723 (+37 ms)Flushing 04cac2e3195e2ed23ca8b3c75b748c09/cf1: closing flushed file at 1733754149728 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@db362df: reopening flushed file at 1733754149799 (+71 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 04cac2e3195e2ed23ca8b3c75b748c09 in 129ms, sequenceid=5, compaction requested=false at 1733754149813 (+14 ms)Writing region close event to WAL at 1733754149815 (+2 ms)Running coprocessor post-close hooks at 1733754149822 (+7 ms)Closed at 1733754149825 (+3 ms) 2024-12-09T14:22:29,826 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 04cac2e3195e2ed23ca8b3c75b748c09 move to f4e784dc7cb5,33819,1733754130225 record at close sequenceid=5 2024-12-09T14:22:29,830 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:29,831 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=CLOSED 2024-12-09T14:22:29,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338 because future has completed 2024-12-09T14:22:29,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T14:22:29,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338 in 323 msec 2024-12-09T14:22:29,845 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE; state=CLOSED, location=f4e784dc7cb5,33819,1733754130225; forceNewPlan=false, retain=false 2024-12-09T14:22:29,996 INFO [f4e784dc7cb5:39231 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:22:29,997 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPENING, regionLocation=f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:30,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE because future has completed 2024-12-09T14:22:30,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,33819,1733754130225}] 2024-12-09T14:22:30,176 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,176 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:30,177 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,177 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:30,177 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,177 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,192 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,203 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf1 2024-12-09T14:22:30,203 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:30,222 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd 2024-12-09T14:22:30,222 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:30,222 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,223 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf2 2024-12-09T14:22:30,224 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:30,224 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:30,225 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,226 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,227 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,229 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,229 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,231 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:22:30,233 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,235 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 04cac2e3195e2ed23ca8b3c75b748c09; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70088663, jitterRate=0.04440246522426605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:22:30,235 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,236 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 04cac2e3195e2ed23ca8b3c75b748c09: Running coprocessor pre-open hook at 1733754150177Writing region info on filesystem at 1733754150177Initializing all the Stores at 1733754150187 (+10 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754150187Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754150190 (+3 ms)Cleaning up temporary data from old regions at 1733754150229 (+39 ms)Running coprocessor post-open hooks at 1733754150235 (+6 ms)Region opened successfully at 1733754150235 2024-12-09T14:22:30,239 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., pid=9, masterSystemTime=1733754150170 2024-12-09T14:22:30,243 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,244 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPEN, openSeqNum=9, regionLocation=f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:30,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,33819,1733754130225 because future has completed 2024-12-09T14:22:30,255 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39231 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f4e784dc7cb5,33819,1733754130225, table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T14:22:30,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T14:22:30,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,33819,1733754130225 in 254 msec 2024-12-09T14:22:30,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE in 771 msec 2024-12-09T14:22:30,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:22:30,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:22:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:47846 deadline: 1733754210312, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=33819 startCode=1733754130225. As of locationSeqNum=5. 2024-12-09T14:22:30,320 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=33819 startCode=1733754130225. As of locationSeqNum=5. 2024-12-09T14:22:30,321 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=33819 startCode=1733754130225. As of locationSeqNum=5. 2024-12-09T14:22:30,321 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,33819,1733754130225, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=33819 startCode=1733754130225. As of locationSeqNum=5. 2024-12-09T14:22:30,438 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:22:30,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:22:30,455 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 04cac2e3195e2ed23ca8b3c75b748c09 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T14:22:30,484 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/d452e3c4f7564872a1d39bed1201fb4e is 29, key is r1/cf1:/1733754150442/DeleteFamily/seqid=0 2024-12-09T14:22:30,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741888_1066 (size=4906) 2024-12-09T14:22:30,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741888_1066 (size=4906) 2024-12-09T14:22:30,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741888_1066 (size=4906) 2024-12-09T14:22:30,519 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:30,528 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:30,552 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf2/af8809e9470f435db07801f0a3110e65 is 29, key is r1/cf2:/1733754150442/DeleteFamily/seqid=0 2024-12-09T14:22:30,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741889_1067 (size=4906) 2024-12-09T14:22:30,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741889_1067 (size=4906) 2024-12-09T14:22:30,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741889_1067 (size=4906) 2024-12-09T14:22:30,577 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf2/af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:30,586 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:30,589 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/d452e3c4f7564872a1d39bed1201fb4e as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:30,598 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:30,598 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T14:22:30,600 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf2/af8809e9470f435db07801f0a3110e65 as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:30,614 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:30,615 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T14:22:30,618 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 04cac2e3195e2ed23ca8b3c75b748c09 in 163ms, sequenceid=12, compaction requested=false 2024-12-09T14:22:30,618 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 04cac2e3195e2ed23ca8b3c75b748c09: 2024-12-09T14:22:30,622 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T14:22:30,624 DEBUG [Time-limited test {}] regionserver.HStore(1541): 04cac2e3195e2ed23ca8b3c75b748c09/cf1 is initiating major compaction (all files) 2024-12-09T14:22:30,624 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:22:30,624 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:30,625 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 04cac2e3195e2ed23ca8b3c75b748c09/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,625 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd, hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e] into tmpdir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp, totalSize=9.5 K 2024-12-09T14:22:30,627 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ef6d12e1fc3749aca8201bca58b6cffd, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733754149472 2024-12-09T14:22:30,628 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d452e3c4f7564872a1d39bed1201fb4e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T14:22:30,660 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 04cac2e3195e2ed23ca8b3c75b748c09#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T14:22:30,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741890_1068 (size=4626) 2024-12-09T14:22:30,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741890_1068 (size=4626) 2024-12-09T14:22:30,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741890_1068 (size=4626) 2024-12-09T14:22:30,698 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf1/e3d28462bbd946629b218cdf26588976 as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/e3d28462bbd946629b218cdf26588976 2024-12-09T14:22:30,723 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 04cac2e3195e2ed23ca8b3c75b748c09/cf1 of 04cac2e3195e2ed23ca8b3c75b748c09 into e3d28462bbd946629b218cdf26588976(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T14:22:30,723 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 04cac2e3195e2ed23ca8b3c75b748c09: 2024-12-09T14:22:30,723 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T14:22:30,723 DEBUG [Time-limited test {}] regionserver.HStore(1541): 04cac2e3195e2ed23ca8b3c75b748c09/cf2 is initiating major compaction (all files) 2024-12-09T14:22:30,724 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:22:30,724 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:22:30,724 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 04cac2e3195e2ed23ca8b3c75b748c09/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,724 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65] into tmpdir=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp, totalSize=4.8 K 2024-12-09T14:22:30,725 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting af8809e9470f435db07801f0a3110e65, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T14:22:30,735 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 04cac2e3195e2ed23ca8b3c75b748c09#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T14:22:30,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741891_1069 (size=4592) 2024-12-09T14:22:30,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741891_1069 (size=4592) 2024-12-09T14:22:30,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741891_1069 (size=4592) 2024-12-09T14:22:30,783 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/.tmp/cf2/d753e1f757c74132991016d95d75e032 as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/d753e1f757c74132991016d95d75e032 2024-12-09T14:22:30,794 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 04cac2e3195e2ed23ca8b3c75b748c09/cf2 of 04cac2e3195e2ed23ca8b3c75b748c09 into d753e1f757c74132991016d95d75e032(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T14:22:30,794 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 04cac2e3195e2ed23ca8b3c75b748c09: 2024-12-09T14:22:30,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=04cac2e3195e2ed23ca8b3c75b748c09, source=f4e784dc7cb5,33819,1733754130225, destination=f4e784dc7cb5,43199,1733754130338, warming up region on f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:30,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=04cac2e3195e2ed23ca8b3c75b748c09, source=f4e784dc7cb5,33819,1733754130225, destination=f4e784dc7cb5,43199,1733754130338, running balancer 2024-12-09T14:22:30,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(7855): Warmup {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE 2024-12-09T14:22:30,807 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE 2024-12-09T14:22:30,810 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=CLOSING, regionLocation=f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:30,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE because future has completed 2024-12-09T14:22:30,814 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T14:22:30,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,33819,1733754130225}] 2024-12-09T14:22:30,814 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,816 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf1 2024-12-09T14:22:30,816 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:30,839 INFO [StoreFileOpener-04cac2e3195e2ed23ca8b3c75b748c09-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:30,840 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:30,847 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/e3d28462bbd946629b218cdf26588976 2024-12-09T14:22:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T14:22:30,859 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd 2024-12-09T14:22:30,859 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:30,860 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,861 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf2 2024-12-09T14:22:30,861 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:30,893 INFO [StoreFileOpener-04cac2e3195e2ed23ca8b3c75b748c09-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:30,893 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:30,902 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/d753e1f757c74132991016d95d75e032 2024-12-09T14:22:30,902 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:30,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43199 {}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754150902Disabling compacts and flushes for region at 1733754150902Disabling writes for close at 1733754150903 (+1 ms)Writing region close event to WAL at 1733754150927 (+24 ms)Closed at 1733754150927 2024-12-09T14:22:30,975 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:30,975 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T14:22:30,975 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:30,975 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,975 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,975 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:30,976 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:30,976 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd, hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e] to archive 2024-12-09T14:22:30,980 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T14:22:30,985 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd to hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/ef6d12e1fc3749aca8201bca58b6cffd 2024-12-09T14:22:30,988 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e to hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/d452e3c4f7564872a1d39bed1201fb4e 2024-12-09T14:22:31,007 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65] to archive 2024-12-09T14:22:31,008 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T14:22:31,015 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65 to hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/af8809e9470f435db07801f0a3110e65 2024-12-09T14:22:31,026 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T14:22:31,027 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,028 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754150975Running coprocessor pre-close hooks at 1733754150975Disabling compacts and flushes for region at 1733754150975Disabling writes for close at 1733754150976 (+1 ms)Writing region close event to WAL at 1733754151017 (+41 ms)Running coprocessor post-close hooks at 1733754151027 (+10 ms)Closed at 1733754151027 2024-12-09T14:22:31,028 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 04cac2e3195e2ed23ca8b3c75b748c09 move to f4e784dc7cb5,43199,1733754130338 record at close sequenceid=12 2024-12-09T14:22:31,035 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,040 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=CLOSED 2024-12-09T14:22:31,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,33819,1733754130225 because future has completed 2024-12-09T14:22:31,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T14:22:31,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,33819,1733754130225 in 250 msec 2024-12-09T14:22:31,081 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE; state=CLOSED, location=f4e784dc7cb5,43199,1733754130338; forceNewPlan=false, retain=false 2024-12-09T14:22:31,232 INFO [f4e784dc7cb5:39231 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:22:31,232 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPENING, regionLocation=f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:31,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE because future has completed 2024-12-09T14:22:31,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338}] 2024-12-09T14:22:31,395 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,395 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:31,396 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,396 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:31,396 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,396 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,398 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,400 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf1 2024-12-09T14:22:31,400 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:31,409 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/e3d28462bbd946629b218cdf26588976 2024-12-09T14:22:31,409 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:31,410 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,411 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf2 2024-12-09T14:22:31,412 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:31,420 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/d753e1f757c74132991016d95d75e032 2024-12-09T14:22:31,420 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:31,421 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,422 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,423 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,424 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,424 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,425 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:22:31,428 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,429 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 04cac2e3195e2ed23ca8b3c75b748c09; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59555922, jitterRate=-0.1125476062297821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:22:31,429 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,430 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 04cac2e3195e2ed23ca8b3c75b748c09: Running coprocessor pre-open hook at 1733754151397Writing region info on filesystem at 1733754151397Initializing all the Stores at 1733754151398 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754151398Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754151398Cleaning up temporary data from old regions at 1733754151424 (+26 ms)Running coprocessor post-open hooks at 1733754151429 (+5 ms)Region opened successfully at 1733754151430 (+1 ms) 2024-12-09T14:22:31,431 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., pid=12, masterSystemTime=1733754151390 2024-12-09T14:22:31,434 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,435 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,435 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPEN, openSeqNum=18, regionLocation=f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:31,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338 because future has completed 2024-12-09T14:22:31,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T14:22:31,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,43199,1733754130338 in 206 msec 2024-12-09T14:22:31,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, REOPEN/MOVE in 651 msec 2024-12-09T14:22:31,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:22:31,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:22:31,469 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server f4e784dc7cb5,43199,1733754130338: testing ***** 2024-12-09T14:22:31,469 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T14:22:31,472 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T14:22:31,474 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T14:22:31,481 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T14:22:31,484 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T14:22:31,499 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 352262720 }, "NonHeapMemoryUsage": { "committed": 172883968, "init": 7667712, "max": -1, "used": 167187552 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f4e784dc7cb5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11076, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 125, "ProcessCallTime_mean": 8, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 3, "ProcessCallTime_75th_percentile": 5, "ProcessCallTime_90th_percentile": 11, "ProcessCallTime_95th_percentile": 42, "ProcessCallTime_98th_percentile": 125, "ProcessCallTime_99th_percentile": 125, "ProcessCallTime_99.9th_percentile": 125, "ProcessCallTime_TimeRangeCount_0-1": 4, "ProcessCallTime_TimeRangeCount_1-3": 15, "ProcessCallTime_TimeRangeCount_3-10": 8, "ProcessCallTime_TimeRangeCount_10-30": 2, "ProcessCallTime_TimeRangeCount_30-100": 1, "ProcessCallTime_TimeRangeCount_100-300": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 125, "TotalCallTime_mean": 9, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 3, "TotalCallTime_75th_percentile": 5, "TotalCallTime_90th_percentile": 11, "TotalCallTime_95th_percentile": 42, "TotalCallTime_98th_percentile": 125, "TotalCallTime_99th_percentile": 125, "TotalCallTime_99.9th_percentile": 125, "TotalCallTime_TimeRangeCount_0-1": 4, "TotalCallTime_TimeRangeCount_1-3": 14, "TotalCallTime_TimeRangeCount_3-10": 9, "TotalCallTime_TimeRangeCount_10-30": 2, "TotalCallTime_TimeRangeCount_30-100": 1, "TotalCallTime_TimeRangeCount_100-300": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5751 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f4e784dc7cb5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49681", "tag.serverName": "f4e784dc7cb5,43199,1733754130338", "tag.clusterId": "1a8cc862-cf33-438f-b5cf-d9e9d16321ec", "tag.Context": "regionserver", "tag.Hostname": "f4e784dc7cb5", "regionCount": 1, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733754130338, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 3, "Put_max": 3, "Put_mean": 3, "Put_25th_percentile": 3, "Put_median": 3, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 129, "FlushTime_max": 129, "FlushTime_mean": 129, "FlushTime_25th_percentile": 129, "FlushTime_median": 129, "FlushTime_75th_percentile": 129, "FlushTime_90th_percentile": 129, "FlushTime_95th_percentile": 129, "FlushTime_98th_percentile": 129, "FlushTime_99th_percentile": 129, "FlushTime_99.9th_percentile": 129, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T14:22:31,504 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39231 {}] master.MasterRpcServices(700): f4e784dc7cb5,43199,1733754130338 reported a fatal error: ***** ABORTING region server f4e784dc7cb5,43199,1733754130338: testing ***** 2024-12-09T14:22:31,506 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f4e784dc7cb5,43199,1733754130338' ***** 2024-12-09T14:22:31,506 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T14:22:31,506 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:22:31,507 INFO [RS:1;f4e784dc7cb5:43199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T14:22:31,507 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:22:31,507 INFO [RS:1;f4e784dc7cb5:43199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T14:22:31,507 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(3091): Received CLOSE for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:35758 deadline: 1733754211507, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=43199 startCode=1733754130338. As of locationSeqNum=12. 2024-12-09T14:22:31,508 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,33819,1733754130225, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,33819,1733754130225, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=43199 startCode=1733754130338. As of locationSeqNum=12. 2024-12-09T14:22:31,508 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,33819,1733754130225, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=43199 startCode=1733754130338. As of locationSeqNum=12. 2024-12-09T14:22:31,508 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,33819,1733754130225, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f4e784dc7cb5 port=43199 startCode=1733754130338. As of locationSeqNum=12. 2024-12-09T14:22:31,509 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(956): aborting server f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:31,509 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:22:31,509 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:31,509 INFO [RS:1;f4e784dc7cb5:43199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f4e784dc7cb5:43199. 2024-12-09T14:22:31,509 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,510 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,510 DEBUG [RS:1;f4e784dc7cb5:43199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:22:31,510 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:31,510 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,510 DEBUG [RS:1;f4e784dc7cb5:43199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:31,510 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:22:31,511 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1325): Online Regions={04cac2e3195e2ed23ca8b3c75b748c09=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.} 2024-12-09T14:22:31,511 DEBUG [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1351): Waiting on 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:31,512 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,512 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754151509Running coprocessor pre-close hooks at 1733754151509Disabling compacts and flushes for region at 1733754151509Disabling writes for close at 1733754151510 (+1 ms)Writing region close event to WAL at 1733754151512 (+2 ms)Running coprocessor post-close hooks at 1733754151512Closed at 1733754151512 2024-12-09T14:22:31,512 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:31,606 INFO [regionserver/f4e784dc7cb5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:22:31,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f4e784dc7cb5,43199,1733754130338 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:31,619 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f4e784dc7cb5,43199,1733754130338 aborting 2024-12-09T14:22:31,619 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f4e784dc7cb5,43199,1733754130338 aborting 2024-12-09T14:22:31,619 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=12 from cache 2024-12-09T14:22:31,712 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(976): stopping server f4e784dc7cb5,43199,1733754130338; all regions closed. 2024-12-09T14:22:31,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741834_1010 (size=1407) 2024-12-09T14:22:31,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741834_1010 (size=1407) 2024-12-09T14:22:31,717 DEBUG [RS:1;f4e784dc7cb5:43199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:31,718 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:22:31,718 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:22:31,718 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.ChoreService(370): Chore service for: regionserver/f4e784dc7cb5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:22:31,719 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:22:31,719 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:22:31,718 INFO [regionserver/f4e784dc7cb5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:22:31,719 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:22:31,719 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:22:31,719 INFO [RS:1;f4e784dc7cb5:43199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43199 2024-12-09T14:22:31,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:22:31,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f4e784dc7cb5,43199,1733754130338 2024-12-09T14:22:31,730 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:22:31,735 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f4e784dc7cb5,43199,1733754130338] 2024-12-09T14:22:31,739 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f4e784dc7cb5,43199,1733754130338 already deleted, retry=false 2024-12-09T14:22:31,739 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f4e784dc7cb5,43199,1733754130338 on f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:31,745 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f4e784dc7cb5,43199,1733754130338, splitWal=true, meta=false 2024-12-09T14:22:31,748 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for f4e784dc7cb5,43199,1733754130338 (carryingMeta=false) f4e784dc7cb5,43199,1733754130338/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7d3723ce[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T14:22:31,749 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f4e784dc7cb5,43199,1733754130338, splitWal=true, meta=false 2024-12-09T14:22:31,752 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): f4e784dc7cb5,43199,1733754130338 had 1 regions 2024-12-09T14:22:31,755 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f4e784dc7cb5,43199,1733754130338, splitWal=true, meta=false, isMeta: false 2024-12-09T14:22:31,763 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting 2024-12-09T14:22:31,764 INFO [PEWorker-5 {}] master.SplitWALManager(105): f4e784dc7cb5,43199,1733754130338 WAL count=1, meta=false 2024-12-09T14:22:31,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f4e784dc7cb5%2C43199%2C1733754130338.1733754131925}] 2024-12-09T14:22:31,776 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:31,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, worker=f4e784dc7cb5,40373,1733754130392}] 2024-12-09T14:22:31,832 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18] 2024-12-09T14:22:31,834 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f4e784dc7cb5:43199 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f4e784dc7cb5/172.17.0.3:43199 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:31,835 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18, error=java.net.ConnectException: Call to address=f4e784dc7cb5:43199 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f4e784dc7cb5/172.17.0.3:43199 2024-12-09T14:22:31,835 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T14:22:31,835 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18 from cache 2024-12-09T14:22:31,836 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address f4e784dc7cb5:43199 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f4e784dc7cb5/172.17.0.3:43199 2024-12-09T14:22:31,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:31,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43199-0x1012b90185b0002, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:31,840 INFO [RS:1;f4e784dc7cb5:43199 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:22:31,840 INFO [RS:1;f4e784dc7cb5:43199 {}] regionserver.HRegionServer(1031): Exiting; stopping=f4e784dc7cb5,43199,1733754130338; zookeeper connection closed. 2024-12-09T14:22:31,841 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67b750e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67b750e7 2024-12-09T14:22:31,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40373 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T14:22:31,968 INFO [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, size=1.4 K (1407bytes) 2024-12-09T14:22:31,968 INFO [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 2024-12-09T14:22:31,969 INFO [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 after 0ms 2024-12-09T14:22:31,973 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:31,973 INFO [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 took 5ms 2024-12-09T14:22:31,983 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 04cac2e3195e2ed23ca8b3c75b748c09: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T14:22:31,983 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 so closing down 2024-12-09T14:22:31,984 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:31,984 INFO [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:31,984 INFO [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, size=1.4 K, length=1407, corrupted=false, cancelled=false 2024-12-09T14:22:31,984 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, journal: Splitting hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, size=1.4 K (1407bytes) at 1733754151968Finishing writing output for hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 so closing down at 1733754151983 (+15 ms)3 split writer threads finished at 1733754151984 (+1 ms)Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, size=1.4 K, length=1407, corrupted=false, cancelled=false at 1733754151984 2024-12-09T14:22:31,984 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 2024-12-09T14:22:31,986 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f4e784dc7cb5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T14:22:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39231 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T14:22:31,995 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting/f4e784dc7cb5%2C43199%2C1733754130338.1733754131925 to hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs 2024-12-09T14:22:31,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T14:22:31,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, worker=f4e784dc7cb5,40373,1733754130392 in 216 msec 2024-12-09T14:22:32,001 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:32,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T14:22:32,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure f4e784dc7cb5%2C43199%2C1733754130338.1733754131925, worker=f4e784dc7cb5,40373,1733754130392 in 236 msec 2024-12-09T14:22:32,007 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting dir is empty, no logs to split. 2024-12-09T14:22:32,008 INFO [PEWorker-5 {}] master.SplitWALManager(105): f4e784dc7cb5,43199,1733754130338 WAL count=0, meta=false 2024-12-09T14:22:32,008 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if f4e784dc7cb5,43199,1733754130338 WAL splitting is done? wals=0, meta=false 2024-12-09T14:22:32,010 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for f4e784dc7cb5,43199,1733754130338 failed, ignore...File hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/WALs/f4e784dc7cb5,43199,1733754130338-splitting does not exist. 2024-12-09T14:22:32,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN}] 2024-12-09T14:22:32,015 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN 2024-12-09T14:22:32,016 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T14:22:32,141 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18] 2024-12-09T14:22:32,142 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to f4e784dc7cb5:43199 this server is in the failed servers list 2024-12-09T14:22:32,142 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f4e784dc7cb5:43199 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f4e784dc7cb5:43199 2024-12-09T14:22:32,143 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f4e784dc7cb5:43199 2024-12-09T14:22:32,143 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,43199,1733754130338, seqNum=18 from cache 2024-12-09T14:22:32,167 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(204): Hosts are {f4e784dc7cb5=0} racks are {/default-rack=0} 2024-12-09T14:22:32,167 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:22:32,167 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:22:32,167 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:22:32,167 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:22:32,167 INFO [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:22:32,167 INFO [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:22:32,167 DEBUG [f4e784dc7cb5:39231 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:22:32,168 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPENING, regionLocation=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:32,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN because future has completed 2024-12-09T14:22:32,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,40373,1733754130392}] 2024-12-09T14:22:32,330 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:32,331 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 04cac2e3195e2ed23ca8b3c75b748c09, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:32,331 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,331 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:32,331 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,331 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,338 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,340 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf1 2024-12-09T14:22:32,340 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,349 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf1/e3d28462bbd946629b218cdf26588976 2024-12-09T14:22:32,349 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,349 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,351 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04cac2e3195e2ed23ca8b3c75b748c09 columnFamilyName cf2 2024-12-09T14:22:32,351 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,375 DEBUG [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/cf2/d753e1f757c74132991016d95d75e032 2024-12-09T14:22:32,376 INFO [StoreOpener-04cac2e3195e2ed23ca8b3c75b748c09-1 {}] regionserver.HStore(327): Store=04cac2e3195e2ed23ca8b3c75b748c09/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,376 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,377 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,379 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,380 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,380 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,381 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:22:32,383 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,385 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 04cac2e3195e2ed23ca8b3c75b748c09; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68804577, jitterRate=0.02526809275150299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:22:32,385 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:32,386 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 04cac2e3195e2ed23ca8b3c75b748c09: Running coprocessor pre-open hook at 1733754152332Writing region info on filesystem at 1733754152332Initializing all the Stores at 1733754152333 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152333Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152334 (+1 ms)Cleaning up temporary data from old regions at 1733754152380 (+46 ms)Running coprocessor post-open hooks at 1733754152385 (+5 ms)Region opened successfully at 1733754152386 (+1 ms) 2024-12-09T14:22:32,391 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., pid=17, masterSystemTime=1733754152325 2024-12-09T14:22:32,394 DEBUG [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:32,394 INFO [RS_OPEN_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:32,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=04cac2e3195e2ed23ca8b3c75b748c09, regionState=OPEN, openSeqNum=18, regionLocation=f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:32,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,40373,1733754130392 because future has completed 2024-12-09T14:22:32,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T14:22:32,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 04cac2e3195e2ed23ca8b3c75b748c09, server=f4e784dc7cb5,40373,1733754130392 in 230 msec 2024-12-09T14:22:32,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T14:22:32,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=04cac2e3195e2ed23ca8b3c75b748c09, ASSIGN in 391 msec 2024-12-09T14:22:32,409 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server f4e784dc7cb5,43199,1733754130338 after splitting done 2024-12-09T14:22:32,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure f4e784dc7cb5,43199,1733754130338, splitWal=true, meta=false in 669 msec 2024-12-09T14:22:32,651 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., hostname=f4e784dc7cb5,40373,1733754130392, seqNum=18] 2024-12-09T14:22:32,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 407), OpenFileDescriptor=1040 (was 1011) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=585 (was 506) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5626 (was 5700) 2024-12-09T14:22:32,673 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-12-09T14:22:32,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1040, MaxFileDescriptor=1048576, SystemLoadAverage=585, ProcessCount=11, AvailableMemoryMB=5626 2024-12-09T14:22:32,690 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-12-09T14:22:32,714 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:32,716 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:32,717 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:32,721 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80802155, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-80802155, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:32,739 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80802155/hregion-80802155.1733754152721, exclude list is [], retry=0 2024-12-09T14:22:32,742 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:32,742 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:32,743 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:32,751 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80802155/hregion-80802155.1733754152721 2024-12-09T14:22:32,755 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:32,755 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 271d73c2191d9606511f0f709598b412, NAME => 'testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:32,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741893_1071 (size=67) 2024-12-09T14:22:32,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741893_1071 (size=67) 2024-12-09T14:22:32,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741893_1071 (size=67) 2024-12-09T14:22:32,768 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:32,770 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,772 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName a 2024-12-09T14:22:32,772 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,773 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,773 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,774 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName b 2024-12-09T14:22:32,775 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,775 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,775 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,777 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName c 2024-12-09T14:22:32,777 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,782 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,786 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,791 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,791 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,792 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:32,794 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,797 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:32,798 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 271d73c2191d9606511f0f709598b412; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68236589, jitterRate=0.016804412007331848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:32,799 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 271d73c2191d9606511f0f709598b412: Writing region info on filesystem at 1733754152768Initializing all the Stores at 1733754152769 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152770 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152770Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152770Cleaning up temporary data from old regions at 1733754152791 (+21 ms)Region opened successfully at 1733754152798 (+7 ms) 2024-12-09T14:22:32,799 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 271d73c2191d9606511f0f709598b412, disabling compactions & flushes 2024-12-09T14:22:32,799 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:32,799 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:32,799 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. after waiting 0 ms 2024-12-09T14:22:32,799 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:32,799 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:32,799 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 271d73c2191d9606511f0f709598b412: Waiting for close lock at 1733754152799Disabling compacts and flushes for region at 1733754152799Disabling writes for close at 1733754152799Writing region close event to WAL at 1733754152799Closed at 1733754152799 2024-12-09T14:22:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741892_1070 (size=95) 2024-12-09T14:22:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741892_1070 (size=95) 2024-12-09T14:22:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741892_1070 (size=95) 2024-12-09T14:22:32,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:32,807 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80802155:(num 1733754152721) 2024-12-09T14:22:32,807 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:32,809 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:32,828 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809, exclude list is [], retry=0 2024-12-09T14:22:32,831 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:32,831 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:32,832 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:32,836 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 2024-12-09T14:22:32,838 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:32,838 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 271d73c2191d9606511f0f709598b412, NAME => 'testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:32,838 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:32,838 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,838 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,840 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,841 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName a 2024-12-09T14:22:32,841 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,842 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,842 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,843 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName b 2024-12-09T14:22:32,843 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,843 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,843 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,844 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName c 2024-12-09T14:22:32,845 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:32,845 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:32,846 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,847 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,850 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,850 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,851 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:32,854 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:32,855 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 271d73c2191d9606511f0f709598b412; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70946638, jitterRate=0.057187288999557495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:32,856 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 271d73c2191d9606511f0f709598b412: Writing region info on filesystem at 1733754152838Initializing all the Stores at 1733754152839 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152839Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152840 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754152840Cleaning up temporary data from old regions at 1733754152850 (+10 ms)Region opened successfully at 1733754152856 (+6 ms) 2024-12-09T14:22:32,902 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 271d73c2191d9606511f0f709598b412 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T14:22:32,933 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/a/41af4dec75774033b45b193f35430f29 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733754152856/Put/seqid=0 2024-12-09T14:22:32,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741895_1073 (size=5958) 2024-12-09T14:22:32,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741895_1073 (size=5958) 2024-12-09T14:22:32,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741895_1073 (size=5958) 2024-12-09T14:22:32,979 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/a/41af4dec75774033b45b193f35430f29 2024-12-09T14:22:33,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/b/cfde884a5659432c8b942dab02643c5c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733754152868/Put/seqid=0 2024-12-09T14:22:33,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741896_1074 (size=5958) 2024-12-09T14:22:33,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741896_1074 (size=5958) 2024-12-09T14:22:33,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741896_1074 (size=5958) 2024-12-09T14:22:33,035 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/b/cfde884a5659432c8b942dab02643c5c 2024-12-09T14:22:33,066 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/c/04b5abcef79d4b6b980b0a9d692f95ae is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733754152887/Put/seqid=0 2024-12-09T14:22:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741897_1075 (size=5958) 2024-12-09T14:22:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741897_1075 (size=5958) 2024-12-09T14:22:33,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741897_1075 (size=5958) 2024-12-09T14:22:33,108 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/c/04b5abcef79d4b6b980b0a9d692f95ae 2024-12-09T14:22:33,116 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/a/41af4dec75774033b45b193f35430f29 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/a/41af4dec75774033b45b193f35430f29 2024-12-09T14:22:33,123 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/a/41af4dec75774033b45b193f35430f29, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T14:22:33,125 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/b/cfde884a5659432c8b942dab02643c5c as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/b/cfde884a5659432c8b942dab02643c5c 2024-12-09T14:22:33,132 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/b/cfde884a5659432c8b942dab02643c5c, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T14:22:33,133 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/c/04b5abcef79d4b6b980b0a9d692f95ae as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/c/04b5abcef79d4b6b980b0a9d692f95ae 2024-12-09T14:22:33,140 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/c/04b5abcef79d4b6b980b0a9d692f95ae, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T14:22:33,142 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 271d73c2191d9606511f0f709598b412 in 239ms, sequenceid=33, compaction requested=false 2024-12-09T14:22:33,142 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 271d73c2191d9606511f0f709598b412: 2024-12-09T14:22:33,142 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 271d73c2191d9606511f0f709598b412, disabling compactions & flushes 2024-12-09T14:22:33,142 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:33,142 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:33,142 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. after waiting 0 ms 2024-12-09T14:22:33,142 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:33,147 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. 2024-12-09T14:22:33,147 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 271d73c2191d9606511f0f709598b412: Waiting for close lock at 1733754153142Disabling compacts and flushes for region at 1733754153142Disabling writes for close at 1733754153142Writing region close event to WAL at 1733754153147 (+5 ms)Closed at 1733754153147 2024-12-09T14:22:33,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741894_1072 (size=3384) 2024-12-09T14:22:33,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741894_1072 (size=3384) 2024-12-09T14:22:33,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741894_1072 (size=3384) 2024-12-09T14:22:33,164 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/b/cfde884a5659432c8b942dab02643c5c to hdfs://localhost:43873/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/b/cfde884a5659432c8b942dab02643c5c 2024-12-09T14:22:33,190 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809, size=3.3 K (3384bytes) 2024-12-09T14:22:33,190 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 2024-12-09T14:22:33,190 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 after 0ms 2024-12-09T14:22:33,193 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:33,194 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 took 5ms 2024-12-09T14:22:33,197 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 so closing down 2024-12-09T14:22:33,197 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:33,200 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733754152809.temp 2024-12-09T14:22:33,201 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000003-wal.1733754152809.temp 2024-12-09T14:22:33,202 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:33,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741898_1076 (size=2944) 2024-12-09T14:22:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741898_1076 (size=2944) 2024-12-09T14:22:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741898_1076 (size=2944) 2024-12-09T14:22:33,215 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000003-wal.1733754152809.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:33,218 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000003-wal.1733754152809.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032 2024-12-09T14:22:33,218 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809, size=3.3 K, length=3384, corrupted=false, cancelled=false 2024-12-09T14:22:33,218 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809, journal: Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809, size=3.3 K (3384bytes) at 1733754153190Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 so closing down at 1733754153197 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000003-wal.1733754152809.temp at 1733754153201 (+4 ms)3 split writer threads finished at 1733754153202 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000003-wal.1733754152809.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733754153215 (+13 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000003-wal.1733754152809.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032 at 1733754153218 (+3 ms)Processed 32 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809, size=3.3 K, length=3384, corrupted=false, cancelled=false at 1733754153218 2024-12-09T14:22:33,221 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754152809 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754152809 2024-12-09T14:22:33,222 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032 2024-12-09T14:22:33,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:33,225 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:33,244 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754153225, exclude list is [], retry=0 2024-12-09T14:22:33,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:33,260 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:33,260 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:33,267 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754153225 2024-12-09T14:22:33,267 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:33,267 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 271d73c2191d9606511f0f709598b412, NAME => 'testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:33,268 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:33,268 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,268 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,270 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,271 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName a 2024-12-09T14:22:33,271 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,282 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/a/41af4dec75774033b45b193f35430f29 2024-12-09T14:22:33,282 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,283 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,284 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName b 2024-12-09T14:22:33,284 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,285 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,285 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,286 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 271d73c2191d9606511f0f709598b412 columnFamilyName c 2024-12-09T14:22:33,286 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,297 DEBUG [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/c/04b5abcef79d4b6b980b0a9d692f95ae 2024-12-09T14:22:33,297 INFO [StoreOpener-271d73c2191d9606511f0f709598b412-1 {}] regionserver.HStore(327): Store=271d73c2191d9606511f0f709598b412/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,297 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,300 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,301 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032 2024-12-09T14:22:33,303 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:33,305 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032 2024-12-09T14:22:33,305 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 271d73c2191d9606511f0f709598b412 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T14:22:33,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/b/a5e2a4cbf7d24ce493b5c23041c80bda is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733754152868/Put/seqid=0 2024-12-09T14:22:33,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741900_1078 (size=5958) 2024-12-09T14:22:33,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741900_1078 (size=5958) 2024-12-09T14:22:33,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741900_1078 (size=5958) 2024-12-09T14:22:33,343 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/b/a5e2a4cbf7d24ce493b5c23041c80bda 2024-12-09T14:22:33,351 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/.tmp/b/a5e2a4cbf7d24ce493b5c23041c80bda as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/b/a5e2a4cbf7d24ce493b5c23041c80bda 2024-12-09T14:22:33,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/b/a5e2a4cbf7d24ce493b5c23041c80bda, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T14:22:33,361 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 271d73c2191d9606511f0f709598b412 in 56ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T14:22:33,363 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/0000000000000000032 2024-12-09T14:22:33,368 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,368 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,369 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:33,371 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 271d73c2191d9606511f0f709598b412 2024-12-09T14:22:33,376 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/271d73c2191d9606511f0f709598b412/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T14:22:33,377 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 271d73c2191d9606511f0f709598b412; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71648744, jitterRate=0.0676494836807251}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:33,378 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 271d73c2191d9606511f0f709598b412: Writing region info on filesystem at 1733754153268Initializing all the Stores at 1733754153269 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153269Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153269Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153270 (+1 ms)Obtaining lock to block concurrent updates at 1733754153305 (+35 ms)Preparing flush snapshotting stores in 271d73c2191d9606511f0f709598b412 at 1733754153305Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733754153306 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733754152715.271d73c2191d9606511f0f709598b412. at 1733754153306Flushing 271d73c2191d9606511f0f709598b412/b: creating writer at 1733754153306Flushing 271d73c2191d9606511f0f709598b412/b: appending metadata at 1733754153326 (+20 ms)Flushing 271d73c2191d9606511f0f709598b412/b: closing flushed file at 1733754153326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16a0582f: reopening flushed file at 1733754153350 (+24 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 271d73c2191d9606511f0f709598b412 in 56ms, sequenceid=32, compaction requested=false; wal=null at 1733754153361 (+11 ms)Cleaning up temporary data from old regions at 1733754153368 (+7 ms)Region opened successfully at 1733754153378 (+10 ms) 2024-12-09T14:22:33,410 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=414 (was 405) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:52934 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:53066 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47098 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47274 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:40158 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:40278 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1040) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=585 (was 585), ProcessCount=11 (was 11), AvailableMemoryMB=5618 (was 5626) 2024-12-09T14:22:33,410 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-12-09T14:22:33,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=585, ProcessCount=11, AvailableMemoryMB=5617 2024-12-09T14:22:33,426 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-12-09T14:22:33,447 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:33,449 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:33,450 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:33,453 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10990241, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-10990241, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:33,471 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10990241/hregion-10990241.1733754153454, exclude list is [], retry=0 2024-12-09T14:22:33,475 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:33,475 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:33,476 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:33,478 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10990241/hregion-10990241.1733754153454 2024-12-09T14:22:33,478 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:33,478 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 90b3f6731e2ae719552fa40056a07b97, NAME => 'testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741902_1080 (size=68) 2024-12-09T14:22:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741902_1080 (size=68) 2024-12-09T14:22:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741902_1080 (size=68) 2024-12-09T14:22:33,516 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:33,522 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,525 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName a 2024-12-09T14:22:33,525 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,525 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,525 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,527 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName b 2024-12-09T14:22:33,527 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,528 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,528 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,531 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName c 2024-12-09T14:22:33,531 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,531 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,532 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,534 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,536 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,537 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:33,538 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,544 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:33,544 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90b3f6731e2ae719552fa40056a07b97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71602336, jitterRate=0.06695795059204102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:33,545 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90b3f6731e2ae719552fa40056a07b97: Writing region info on filesystem at 1733754153516Initializing all the Stores at 1733754153517 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153517Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153522 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153522Cleaning up temporary data from old regions at 1733754153536 (+14 ms)Region opened successfully at 1733754153545 (+9 ms) 2024-12-09T14:22:33,545 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 90b3f6731e2ae719552fa40056a07b97, disabling compactions & flushes 2024-12-09T14:22:33,545 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:33,546 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:33,546 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. after waiting 0 ms 2024-12-09T14:22:33,546 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:33,546 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:33,546 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 90b3f6731e2ae719552fa40056a07b97: Waiting for close lock at 1733754153545Disabling compacts and flushes for region at 1733754153545Disabling writes for close at 1733754153546 (+1 ms)Writing region close event to WAL at 1733754153546Closed at 1733754153546 2024-12-09T14:22:33,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741901_1079 (size=95) 2024-12-09T14:22:33,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741901_1079 (size=95) 2024-12-09T14:22:33,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741901_1079 (size=95) 2024-12-09T14:22:33,553 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:33,553 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10990241:(num 1733754153454) 2024-12-09T14:22:33,554 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:33,556 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:33,574 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556, exclude list is [], retry=0 2024-12-09T14:22:33,592 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:33,592 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:33,593 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:33,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 2024-12-09T14:22:33,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:33,680 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 90b3f6731e2ae719552fa40056a07b97, NAME => 'testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:33,683 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,683 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:33,683 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,683 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,686 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,687 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName a 2024-12-09T14:22:33,687 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,688 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,688 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,689 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName b 2024-12-09T14:22:33,690 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,690 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,690 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,691 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName c 2024-12-09T14:22:33,691 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:33,692 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:33,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,695 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,696 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,696 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,697 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:33,699 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,700 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90b3f6731e2ae719552fa40056a07b97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69120920, jitterRate=0.02998197078704834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:33,701 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:33,701 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90b3f6731e2ae719552fa40056a07b97: Running coprocessor pre-open hook at 1733754153684Writing region info on filesystem at 1733754153684Initializing all the Stores at 1733754153685 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153685Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153685Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754153685Cleaning up temporary data from old regions at 1733754153696 (+11 ms)Running coprocessor post-open hooks at 1733754153701 (+5 ms)Region opened successfully at 1733754153701 2024-12-09T14:22:33,721 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 90b3f6731e2ae719552fa40056a07b97 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T14:22:33,722 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:34,723 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:34,767 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:22:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741834_1010 (size=1407) 2024-12-09T14:22:35,724 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:36,725 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:37,726 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:38,347 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T14:22:38,726 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:39,727 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:39,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T14:22:39,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:39,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T14:22:39,766 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:40,728 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:41,729 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:42,730 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 90b3f6731e2ae719552fa40056a07b97/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:42,731 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 90b3f6731e2ae719552fa40056a07b97: 2024-12-09T14:22:42,731 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 90b3f6731e2ae719552fa40056a07b97: 2024-12-09T14:22:42,750 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T14:22:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 90b3f6731e2ae719552fa40056a07b97, disabling compactions & flushes 2024-12-09T14:22:42,750 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. after waiting 0 ms 2024-12-09T14:22:42,750 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:42,763 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:42,763 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. 2024-12-09T14:22:42,763 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 90b3f6731e2ae719552fa40056a07b97: Waiting for close lock at 1733754162750Running coprocessor pre-close hooks at 1733754162750Disabling compacts and flushes for region at 1733754162750Disabling writes for close at 1733754162750Writing region close event to WAL at 1733754162763 (+13 ms)Running coprocessor post-close hooks at 1733754162763Closed at 1733754162763 2024-12-09T14:22:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741903_1081 (size=2691) 2024-12-09T14:22:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741903_1081 (size=2691) 2024-12-09T14:22:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741903_1081 (size=2691) 2024-12-09T14:22:42,790 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556, size=2.6 K (2691bytes) 2024-12-09T14:22:42,790 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 2024-12-09T14:22:42,790 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 after 0ms 2024-12-09T14:22:42,793 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:42,794 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 took 5ms 2024-12-09T14:22:42,797 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 so closing down 2024-12-09T14:22:42,797 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:42,798 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733754153556.temp 2024-12-09T14:22:42,803 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000004-wal.1733754153556.temp 2024-12-09T14:22:42,803 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:42,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741904_1082 (size=2094) 2024-12-09T14:22:42,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741904_1082 (size=2094) 2024-12-09T14:22:42,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741904_1082 (size=2094) 2024-12-09T14:22:42,814 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000004-wal.1733754153556.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:42,816 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000004-wal.1733754153556.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026 2024-12-09T14:22:42,816 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 22 ms; skipped=3; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-12-09T14:22:42,816 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556, journal: Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556, size=2.6 K (2691bytes) at 1733754162790Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 so closing down at 1733754162797 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000004-wal.1733754153556.temp at 1733754162803 (+6 ms)3 split writer threads finished at 1733754162803Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000004-wal.1733754153556.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733754162815 (+12 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000004-wal.1733754153556.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026 at 1733754162816 (+1 ms)Processed 23 edits across 1 Regions in 22 ms; skipped=3; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1733754162816 2024-12-09T14:22:42,819 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754153556 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754153556 2024-12-09T14:22:42,820 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026 2024-12-09T14:22:42,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:42,823 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:42,841 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754162823, exclude list is [], retry=0 2024-12-09T14:22:42,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:42,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:42,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:42,846 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754162823 2024-12-09T14:22:42,847 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:42,847 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 90b3f6731e2ae719552fa40056a07b97, NAME => 'testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:42,848 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,848 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:42,848 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,848 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,850 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,851 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName a 2024-12-09T14:22:42,851 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:42,852 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:42,852 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,853 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName b 2024-12-09T14:22:42,853 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:42,854 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:42,854 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,855 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b3f6731e2ae719552fa40056a07b97 columnFamilyName c 2024-12-09T14:22:42,855 DEBUG [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:42,856 INFO [StoreOpener-90b3f6731e2ae719552fa40056a07b97-1 {}] regionserver.HStore(327): Store=90b3f6731e2ae719552fa40056a07b97/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:42,856 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,857 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,859 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:42,860 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026 2024-12-09T14:22:42,863 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:42,864 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026 2024-12-09T14:22:42,865 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 90b3f6731e2ae719552fa40056a07b97 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T14:22:42,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/a/f8e461d4e10847b6b23fad3f2cdeb9e1 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733754162738/Put/seqid=0 2024-12-09T14:22:42,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741906_1084 (size=5523) 2024-12-09T14:22:42,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741906_1084 (size=5523) 2024-12-09T14:22:42,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741906_1084 (size=5523) 2024-12-09T14:22:42,907 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/a/f8e461d4e10847b6b23fad3f2cdeb9e1 2024-12-09T14:22:42,933 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/b/2eac104dcb2845bb87191bf299ba54c5 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733754162732/Put/seqid=0 2024-12-09T14:22:42,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741907_1085 (size=5524) 2024-12-09T14:22:42,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741907_1085 (size=5524) 2024-12-09T14:22:42,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741907_1085 (size=5524) 2024-12-09T14:22:42,942 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/b/2eac104dcb2845bb87191bf299ba54c5 2024-12-09T14:22:42,975 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/c/276d2c87e0344ffe87592d494ca937f5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733754162735/Put/seqid=0 2024-12-09T14:22:42,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741908_1086 (size=5457) 2024-12-09T14:22:42,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741908_1086 (size=5457) 2024-12-09T14:22:42,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741908_1086 (size=5457) 2024-12-09T14:22:42,994 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/c/276d2c87e0344ffe87592d494ca937f5 2024-12-09T14:22:43,003 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/a/f8e461d4e10847b6b23fad3f2cdeb9e1 as hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/a/f8e461d4e10847b6b23fad3f2cdeb9e1 2024-12-09T14:22:43,009 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/a/f8e461d4e10847b6b23fad3f2cdeb9e1, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T14:22:43,011 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/b/2eac104dcb2845bb87191bf299ba54c5 as hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/b/2eac104dcb2845bb87191bf299ba54c5 2024-12-09T14:22:43,017 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/b/2eac104dcb2845bb87191bf299ba54c5, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T14:22:43,019 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/.tmp/c/276d2c87e0344ffe87592d494ca937f5 as hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/c/276d2c87e0344ffe87592d494ca937f5 2024-12-09T14:22:43,024 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/c/276d2c87e0344ffe87592d494ca937f5, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T14:22:43,025 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 90b3f6731e2ae719552fa40056a07b97 in 160ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T14:22:43,025 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/0000000000000000026 2024-12-09T14:22:43,028 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:43,028 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:43,028 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:43,030 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:43,033 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsAfterAbortingFlush/90b3f6731e2ae719552fa40056a07b97/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T14:22:43,034 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 90b3f6731e2ae719552fa40056a07b97; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61841263, jitterRate=-0.07849337160587311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:43,034 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90b3f6731e2ae719552fa40056a07b97 2024-12-09T14:22:43,035 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 90b3f6731e2ae719552fa40056a07b97: Running coprocessor pre-open hook at 1733754162848Writing region info on filesystem at 1733754162848Initializing all the Stores at 1733754162849 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754162849Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754162850 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754162850Obtaining lock to block concurrent updates at 1733754162865 (+15 ms)Preparing flush snapshotting stores in 90b3f6731e2ae719552fa40056a07b97 at 1733754162865Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733754162865Flushing stores of testReplayEditsAfterAbortingFlush,,1733754153448.90b3f6731e2ae719552fa40056a07b97. at 1733754162865Flushing 90b3f6731e2ae719552fa40056a07b97/a: creating writer at 1733754162865Flushing 90b3f6731e2ae719552fa40056a07b97/a: appending metadata at 1733754162886 (+21 ms)Flushing 90b3f6731e2ae719552fa40056a07b97/a: closing flushed file at 1733754162886Flushing 90b3f6731e2ae719552fa40056a07b97/b: creating writer at 1733754162914 (+28 ms)Flushing 90b3f6731e2ae719552fa40056a07b97/b: appending metadata at 1733754162933 (+19 ms)Flushing 90b3f6731e2ae719552fa40056a07b97/b: closing flushed file at 1733754162933Flushing 90b3f6731e2ae719552fa40056a07b97/c: creating writer at 1733754162951 (+18 ms)Flushing 90b3f6731e2ae719552fa40056a07b97/c: appending metadata at 1733754162974 (+23 ms)Flushing 90b3f6731e2ae719552fa40056a07b97/c: closing flushed file at 1733754162974Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4210d1d7: reopening flushed file at 1733754163002 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7717e50f: reopening flushed file at 1733754163010 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6765501b: reopening flushed file at 1733754163017 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 90b3f6731e2ae719552fa40056a07b97 in 160ms, sequenceid=26, compaction requested=false; wal=null at 1733754163025 (+8 ms)Cleaning up temporary data from old regions at 1733754163028 (+3 ms)Running coprocessor post-open hooks at 1733754163034 (+6 ms)Region opened successfully at 1733754163035 (+1 ms) 2024-12-09T14:22:43,060 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=418 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:35448 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47268 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47278 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:48068 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:48046 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:35464 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 585), ProcessCount=11 (was 11), AvailableMemoryMB=5516 (was 5617) 2024-12-09T14:22:43,060 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-09T14:22:43,076 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=5516 2024-12-09T14:22:43,077 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-09T14:22:43,098 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:43,100 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:43,101 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:43,104 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74649087, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-74649087, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:43,122 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74649087/hregion-74649087.1733754163105, exclude list is [], retry=0 2024-12-09T14:22:43,125 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:43,126 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:43,126 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:43,128 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74649087/hregion-74649087.1733754163105 2024-12-09T14:22:43,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:43,129 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7a5e6068cf3d32a64accb73566f2ce64, NAME => 'testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:43,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741910_1088 (size=61) 2024-12-09T14:22:43,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741910_1088 (size=61) 2024-12-09T14:22:43,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741910_1088 (size=61) 2024-12-09T14:22:43,143 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:43,146 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,148 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a5e6068cf3d32a64accb73566f2ce64 columnFamilyName a 2024-12-09T14:22:43,148 DEBUG [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:43,149 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(327): Store=7a5e6068cf3d32a64accb73566f2ce64/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:43,149 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,150 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,151 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,151 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,151 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,153 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,156 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:43,157 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7a5e6068cf3d32a64accb73566f2ce64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68593731, jitterRate=0.022126242518424988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:43,157 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7a5e6068cf3d32a64accb73566f2ce64: Writing region info on filesystem at 1733754163143Initializing all the Stores at 1733754163144 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754163144Cleaning up temporary data from old regions at 1733754163151 (+7 ms)Region opened successfully at 1733754163157 (+6 ms) 2024-12-09T14:22:43,158 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7a5e6068cf3d32a64accb73566f2ce64, disabling compactions & flushes 2024-12-09T14:22:43,158 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,158 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,158 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. after waiting 0 ms 2024-12-09T14:22:43,158 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,158 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,158 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7a5e6068cf3d32a64accb73566f2ce64: Waiting for close lock at 1733754163158Disabling compacts and flushes for region at 1733754163158Disabling writes for close at 1733754163158Writing region close event to WAL at 1733754163158Closed at 1733754163158 2024-12-09T14:22:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741909_1087 (size=95) 2024-12-09T14:22:43,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741909_1087 (size=95) 2024-12-09T14:22:43,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741909_1087 (size=95) 2024-12-09T14:22:43,165 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:43,165 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74649087:(num 1733754163105) 2024-12-09T14:22:43,165 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:43,167 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:43,186 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167, exclude list is [], retry=0 2024-12-09T14:22:43,189 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:43,189 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:43,190 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:43,203 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 2024-12-09T14:22:43,203 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:43,203 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a5e6068cf3d32a64accb73566f2ce64, NAME => 'testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:43,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:43,204 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,204 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,206 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,207 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a5e6068cf3d32a64accb73566f2ce64 columnFamilyName a 2024-12-09T14:22:43,207 DEBUG [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:43,208 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(327): Store=7a5e6068cf3d32a64accb73566f2ce64/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:43,208 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,212 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,214 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,216 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7a5e6068cf3d32a64accb73566f2ce64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69248087, jitterRate=0.031876906752586365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:43,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7a5e6068cf3d32a64accb73566f2ce64: Writing region info on filesystem at 1733754163204Initializing all the Stores at 1733754163205 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754163205Cleaning up temporary data from old regions at 1733754163212 (+7 ms)Region opened successfully at 1733754163216 (+4 ms) 2024-12-09T14:22:43,232 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7a5e6068cf3d32a64accb73566f2ce64, disabling compactions & flushes 2024-12-09T14:22:43,232 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,233 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,233 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. after waiting 0 ms 2024-12-09T14:22:43,233 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,235 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,235 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. 2024-12-09T14:22:43,235 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7a5e6068cf3d32a64accb73566f2ce64: Waiting for close lock at 1733754163232Disabling compacts and flushes for region at 1733754163232Disabling writes for close at 1733754163233 (+1 ms)Writing region close event to WAL at 1733754163235 (+2 ms)Closed at 1733754163235 2024-12-09T14:22:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741911_1089 (size=1050) 2024-12-09T14:22:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741911_1089 (size=1050) 2024-12-09T14:22:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741911_1089 (size=1050) 2024-12-09T14:22:43,294 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167, size=1.0 K (1050bytes) 2024-12-09T14:22:43,294 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 2024-12-09T14:22:43,296 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 after 2ms 2024-12-09T14:22:43,308 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:43,309 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 took 14ms 2024-12-09T14:22:43,315 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 so closing down 2024-12-09T14:22:43,315 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:43,345 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733754163167.temp 2024-12-09T14:22:43,354 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000003-wal.1733754163167.temp 2024-12-09T14:22:43,355 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:43,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741912_1090 (size=1050) 2024-12-09T14:22:43,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741912_1090 (size=1050) 2024-12-09T14:22:43,409 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000003-wal.1733754163167.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:43,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741912_1090 (size=1050) 2024-12-09T14:22:43,414 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000003-wal.1733754163167.temp to hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012 2024-12-09T14:22:43,414 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 101 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-12-09T14:22:43,414 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167, journal: Splitting hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167, size=1.0 K (1050bytes) at 1733754163294Finishing writing output for hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 so closing down at 1733754163315 (+21 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000003-wal.1733754163167.temp at 1733754163354 (+39 ms)3 split writer threads finished at 1733754163355 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000003-wal.1733754163167.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733754163409 (+54 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000003-wal.1733754163167.temp to hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012 at 1733754163414 (+5 ms)Processed 10 edits across 1 Regions in 101 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1733754163414 2024-12-09T14:22:43,417 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163167 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754163167 2024-12-09T14:22:43,419 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012 2024-12-09T14:22:43,424 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:43,809 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:43,811 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:43,828 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163812, exclude list is [], retry=0 2024-12-09T14:22:43,831 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:43,831 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:43,833 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:43,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163812 2024-12-09T14:22:43,847 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:43,847 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a5e6068cf3d32a64accb73566f2ce64, NAME => 'testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:43,847 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:43,847 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,847 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,850 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,853 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a5e6068cf3d32a64accb73566f2ce64 columnFamilyName a 2024-12-09T14:22:43,853 DEBUG [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:43,854 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(327): Store=7a5e6068cf3d32a64accb73566f2ce64/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:43,854 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,858 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:43,859 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012 2024-12-09T14:22:43,868 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:43,869 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012 2024-12-09T14:22:43,869 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7a5e6068cf3d32a64accb73566f2ce64 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T14:22:43,898 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/.tmp/a/0dfadc9aaf77417b98c95c0622340d7f is 79, key is testDatalossWhenInputError/a:x0/1733754163217/Put/seqid=0 2024-12-09T14:22:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741914_1092 (size=5808) 2024-12-09T14:22:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741914_1092 (size=5808) 2024-12-09T14:22:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741914_1092 (size=5808) 2024-12-09T14:22:44,327 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/.tmp/a/0dfadc9aaf77417b98c95c0622340d7f 2024-12-09T14:22:44,341 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/.tmp/a/0dfadc9aaf77417b98c95c0622340d7f as hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/a/0dfadc9aaf77417b98c95c0622340d7f 2024-12-09T14:22:44,351 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/a/0dfadc9aaf77417b98c95c0622340d7f, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T14:22:44,352 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7a5e6068cf3d32a64accb73566f2ce64 in 483ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T14:22:44,353 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/0000000000000000012 2024-12-09T14:22:44,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,359 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,362 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T14:22:44,363 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7a5e6068cf3d32a64accb73566f2ce64; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63457890, jitterRate=-0.05440375208854675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:44,364 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7a5e6068cf3d32a64accb73566f2ce64: Writing region info on filesystem at 1733754163847Initializing all the Stores at 1733754163850 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754163850Obtaining lock to block concurrent updates at 1733754163870 (+20 ms)Preparing flush snapshotting stores in 7a5e6068cf3d32a64accb73566f2ce64 at 1733754163870Finished memstore snapshotting testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733754163870Flushing stores of testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64. at 1733754163870Flushing 7a5e6068cf3d32a64accb73566f2ce64/a: creating writer at 1733754163870Flushing 7a5e6068cf3d32a64accb73566f2ce64/a: appending metadata at 1733754163897 (+27 ms)Flushing 7a5e6068cf3d32a64accb73566f2ce64/a: closing flushed file at 1733754163897Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19d62ac3: reopening flushed file at 1733754164339 (+442 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7a5e6068cf3d32a64accb73566f2ce64 in 483ms, sequenceid=12, compaction requested=false; wal=null at 1733754164352 (+13 ms)Cleaning up temporary data from old regions at 1733754164355 (+3 ms)Region opened successfully at 1733754164364 (+9 ms) 2024-12-09T14:22:44,369 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a5e6068cf3d32a64accb73566f2ce64, NAME => 'testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:44,369 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733754163098.7a5e6068cf3d32a64accb73566f2ce64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:44,369 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,369 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,371 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,372 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a5e6068cf3d32a64accb73566f2ce64 columnFamilyName a 2024-12-09T14:22:44,372 DEBUG [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:44,380 DEBUG [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/a/0dfadc9aaf77417b98c95c0622340d7f 2024-12-09T14:22:44,380 INFO [StoreOpener-7a5e6068cf3d32a64accb73566f2ce64-1 {}] regionserver.HStore(327): Store=7a5e6068cf3d32a64accb73566f2ce64/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:44,380 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,384 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,385 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,385 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7a5e6068cf3d32a64accb73566f2ce64 2024-12-09T14:22:44,390 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testDatalossWhenInputError/7a5e6068cf3d32a64accb73566f2ce64/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T14:22:44,391 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7a5e6068cf3d32a64accb73566f2ce64; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66041907, jitterRate=-0.015898898243904114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:22:44,391 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7a5e6068cf3d32a64accb73566f2ce64: Writing region info on filesystem at 1733754164369Initializing all the Stores at 1733754164370 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754164370Cleaning up temporary data from old regions at 1733754164385 (+15 ms)Region opened successfully at 1733754164391 (+6 ms) 2024-12-09T14:22:44,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47268 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-728790427-172.17.0.3-1733754125568:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:48068 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47334 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:48134 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:35464 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:35552 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 572), ProcessCount=11 (was 11), AvailableMemoryMB=5491 (was 5516) 2024-12-09T14:22:44,414 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-09T14:22:44,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=5490 2024-12-09T14:22:44,430 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-09T14:22:44,454 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:44,457 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:44,458 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:44,469 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-96843800, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-96843800, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:44,490 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-96843800/hregion-96843800.1733754164470, exclude list is [], retry=0 2024-12-09T14:22:44,499 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:44,499 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:44,510 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:44,531 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-96843800/hregion-96843800.1733754164470 2024-12-09T14:22:44,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:44,531 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee787dff136e27b2ef1ba5db3359e9df, NAME => 'testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:44,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741916_1094 (size=63) 2024-12-09T14:22:44,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741916_1094 (size=63) 2024-12-09T14:22:44,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741916_1094 (size=63) 2024-12-09T14:22:44,974 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:44,976 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,978 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName a 2024-12-09T14:22:44,978 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:44,979 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:44,979 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,980 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName b 2024-12-09T14:22:44,980 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:44,981 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:44,981 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,982 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName c 2024-12-09T14:22:44,982 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:44,983 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:44,983 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,984 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,984 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,986 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,986 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,986 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:44,988 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:44,991 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:44,991 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee787dff136e27b2ef1ba5db3359e9df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64624390, jitterRate=-0.03702154755592346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:44,992 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee787dff136e27b2ef1ba5db3359e9df: Writing region info on filesystem at 1733754164975Initializing all the Stores at 1733754164976 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754164976Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754164976Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754164976Cleaning up temporary data from old regions at 1733754164986 (+10 ms)Region opened successfully at 1733754164992 (+6 ms) 2024-12-09T14:22:44,992 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee787dff136e27b2ef1ba5db3359e9df, disabling compactions & flushes 2024-12-09T14:22:44,992 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:44,992 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:44,992 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. after waiting 0 ms 2024-12-09T14:22:44,992 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:44,993 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:44,993 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee787dff136e27b2ef1ba5db3359e9df: Waiting for close lock at 1733754164992Disabling compacts and flushes for region at 1733754164992Disabling writes for close at 1733754164992Writing region close event to WAL at 1733754164993 (+1 ms)Closed at 1733754164993 2024-12-09T14:22:44,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741915_1093 (size=95) 2024-12-09T14:22:44,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741915_1093 (size=95) 2024-12-09T14:22:44,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741915_1093 (size=95) 2024-12-09T14:22:44,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:44,999 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-96843800:(num 1733754164470) 2024-12-09T14:22:45,000 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:45,002 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:45,020 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002, exclude list is [], retry=0 2024-12-09T14:22:45,023 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:45,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:45,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:45,027 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 2024-12-09T14:22:45,029 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:45,029 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee787dff136e27b2ef1ba5db3359e9df, NAME => 'testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:45,029 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:45,029 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,029 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,031 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,032 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName a 2024-12-09T14:22:45,032 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:45,032 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:45,033 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,034 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName b 2024-12-09T14:22:45,034 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:45,034 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:45,034 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,035 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName c 2024-12-09T14:22:45,035 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:45,036 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:45,036 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,039 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,039 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,040 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:45,042 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:45,043 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee787dff136e27b2ef1ba5db3359e9df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62751383, jitterRate=-0.06493152678012848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:45,044 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee787dff136e27b2ef1ba5db3359e9df: Writing region info on filesystem at 1733754165029Initializing all the Stores at 1733754165030 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754165030Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754165031 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754165031Cleaning up temporary data from old regions at 1733754165039 (+8 ms)Region opened successfully at 1733754165044 (+5 ms) 2024-12-09T14:22:45,050 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733754165050/Put/seqid=0 2024-12-09T14:22:45,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741918_1096 (size=4875) 2024-12-09T14:22:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741918_1096 (size=4875) 2024-12-09T14:22:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741918_1096 (size=4875) 2024-12-09T14:22:45,065 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733754165065/Put/seqid=0 2024-12-09T14:22:45,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741919_1097 (size=4875) 2024-12-09T14:22:45,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741919_1097 (size=4875) 2024-12-09T14:22:45,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741919_1097 (size=4875) 2024-12-09T14:22:45,075 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733754165075/Put/seqid=0 2024-12-09T14:22:45,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741920_1098 (size=4875) 2024-12-09T14:22:45,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741920_1098 (size=4875) 2024-12-09T14:22:45,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741920_1098 (size=4875) 2024-12-09T14:22:45,085 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in ee787dff136e27b2ef1ba5db3359e9df/a 2024-12-09T14:22:45,089 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T14:22:45,089 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:22:45,089 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in ee787dff136e27b2ef1ba5db3359e9df/a 2024-12-09T14:22:45,094 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T14:22:45,094 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:22:45,094 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in ee787dff136e27b2ef1ba5db3359e9df/a 2024-12-09T14:22:45,098 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T14:22:45,098 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:22:45,098 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee787dff136e27b2ef1ba5db3359e9df 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T14:22:45,119 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/.tmp/a/92c5578b19784bfa9fd1c032e533a6f1 is 55, key is testCompactedBulkLoadedFiles/a:a/1733754165045/Put/seqid=0 2024-12-09T14:22:45,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741921_1099 (size=5107) 2024-12-09T14:22:45,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741921_1099 (size=5107) 2024-12-09T14:22:45,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741921_1099 (size=5107) 2024-12-09T14:22:45,128 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/.tmp/a/92c5578b19784bfa9fd1c032e533a6f1 2024-12-09T14:22:45,136 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/.tmp/a/92c5578b19784bfa9fd1c032e533a6f1 as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1 2024-12-09T14:22:45,143 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T14:22:45,144 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for ee787dff136e27b2ef1ba5db3359e9df in 46ms, sequenceid=4, compaction requested=false 2024-12-09T14:22:45,145 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ee787dff136e27b2ef1ba5db3359e9df: 2024-12-09T14:22:45,146 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ 2024-12-09T14:22:45,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ 2024-12-09T14:22:45,150 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ 2024-12-09T14:22:45,150 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile0 into ee787dff136e27b2ef1ba5db3359e9df/a as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ - updating store file list. 2024-12-09T14:22:45,155 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ced8c13e42b547d2baa59be106051db7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:45,156 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ into ee787dff136e27b2ef1ba5db3359e9df/a 2024-12-09T14:22:45,156 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile0 into ee787dff136e27b2ef1ba5db3359e9df/a (new location: hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_) 2024-12-09T14:22:45,157 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile1 into ee787dff136e27b2ef1ba5db3359e9df/a as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ - updating store file list. 2024-12-09T14:22:45,162 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:45,162 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ into ee787dff136e27b2ef1ba5db3359e9df/a 2024-12-09T14:22:45,162 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile1 into ee787dff136e27b2ef1ba5db3359e9df/a (new location: hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_) 2024-12-09T14:22:45,164 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile2 into ee787dff136e27b2ef1ba5db3359e9df/a as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ - updating store file list. 2024-12-09T14:22:45,169 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fcbc5dde2de344759d16146303cd401a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:45,169 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ into ee787dff136e27b2ef1ba5db3359e9df/a 2024-12-09T14:22:45,169 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43873/hbase/testCompactedBulkLoadedFiles/hfile2 into ee787dff136e27b2ef1ba5db3359e9df/a (new location: hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_) 2024-12-09T14:22:45,178 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T14:22:45,178 DEBUG [Time-limited test {}] regionserver.HStore(1541): ee787dff136e27b2ef1ba5db3359e9df/a is initiating major compaction (all files) 2024-12-09T14:22:45,178 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ee787dff136e27b2ef1ba5db3359e9df/a in testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:45,178 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_] into tmpdir=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/.tmp, totalSize=19.3 K 2024-12-09T14:22:45,179 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 92c5578b19784bfa9fd1c032e533a6f1, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733754165045 2024-12-09T14:22:45,179 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ced8c13e42b547d2baa59be106051db7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T14:22:45,180 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T14:22:45,180 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fcbc5dde2de344759d16146303cd401a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T14:22:45,194 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/.tmp/a/854235b7133243f0961649381168a244 is 55, key is testCompactedBulkLoadedFiles/a:a/1733754165045/Put/seqid=0 2024-12-09T14:22:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741922_1100 (size=6154) 2024-12-09T14:22:45,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741922_1100 (size=6154) 2024-12-09T14:22:45,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741922_1100 (size=6154) 2024-12-09T14:22:45,209 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/.tmp/a/854235b7133243f0961649381168a244 as hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/854235b7133243f0961649381168a244 2024-12-09T14:22:45,216 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in ee787dff136e27b2ef1ba5db3359e9df/a of ee787dff136e27b2ef1ba5db3359e9df into 854235b7133243f0961649381168a244(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T14:22:45,217 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ee787dff136e27b2ef1ba5db3359e9df: 2024-12-09T14:22:45,217 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T14:22:45,217 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T14:22:45,260 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002, size=0 (0bytes) 2024-12-09T14:22:45,260 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 might be still open, length is 0 2024-12-09T14:22:45,260 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 2024-12-09T14:22:45,261 WARN [IPC Server handler 2 on default port 43873 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T14:22:45,261 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 after 1ms 2024-12-09T14:22:47,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:48174 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48174 dst: /127.0.0.1:34769 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34769 remote=/127.0.0.1:48174]. Total timeout mills is 60000, 58134 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:47,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:35584 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35501:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35584 dst: /127.0.0.1:35501 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:47,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:47374 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34887:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47374 dst: /127.0.0.1:34887 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741917_1101 (size=1170) 2024-12-09T14:22:47,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741917_1101 (size=1170) 2024-12-09T14:22:48,296 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:22:49,262 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 after 4002ms 2024-12-09T14:22:49,265 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:49,266 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 took 4005ms 2024-12-09T14:22:49,268 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002; continuing. 2024-12-09T14:22:49,268 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 so closing down 2024-12-09T14:22:49,268 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:49,270 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733754165002.temp 2024-12-09T14:22:49,271 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000003-wal.1733754165002.temp 2024-12-09T14:22:49,272 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741923_1102 (size=547) 2024-12-09T14:22:49,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741923_1102 (size=547) 2024-12-09T14:22:49,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741923_1102 (size=547) 2024-12-09T14:22:49,281 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000003-wal.1733754165002.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:49,283 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000003-wal.1733754165002.temp to hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008 2024-12-09T14:22:49,283 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:22:49,283 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002, journal: Splitting hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002, size=0 (0bytes) at 1733754165260Finishing writing output for hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 so closing down at 1733754169268 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000003-wal.1733754165002.temp at 1733754169271 (+3 ms)3 split writer threads finished at 1733754169272 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000003-wal.1733754165002.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733754169281 (+9 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000003-wal.1733754165002.temp to hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008 at 1733754169283 (+2 ms)Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002, size=0, length=0, corrupted=false, cancelled=false at 1733754169283 2024-12-09T14:22:49,285 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754165002 2024-12-09T14:22:49,286 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008 2024-12-09T14:22:49,286 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:49,288 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:49,307 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754169288, exclude list is [], retry=0 2024-12-09T14:22:49,310 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:49,310 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:49,310 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:49,312 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754169288 2024-12-09T14:22:49,312 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:49,313 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee787dff136e27b2ef1ba5db3359e9df, NAME => 'testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:49,313 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:49,313 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,313 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,315 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,316 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName a 2024-12-09T14:22:49,316 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,326 DEBUG [StoreFileOpener-ee787dff136e27b2ef1ba5db3359e9df-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:49,326 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ 2024-12-09T14:22:49,332 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/854235b7133243f0961649381168a244 2024-12-09T14:22:49,336 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1 2024-12-09T14:22:49,340 DEBUG [StoreFileOpener-ee787dff136e27b2ef1ba5db3359e9df-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ced8c13e42b547d2baa59be106051db7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:49,340 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ 2024-12-09T14:22:49,344 DEBUG [StoreFileOpener-ee787dff136e27b2ef1ba5db3359e9df-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fcbc5dde2de344759d16146303cd401a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:22:49,344 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ 2024-12-09T14:22:49,344 WARN [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1e7433c7 2024-12-09T14:22:49,344 WARN [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1e7433c7 2024-12-09T14:22:49,344 WARN [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1e7433c7 2024-12-09T14:22:49,345 WARN [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1e7433c7 2024-12-09T14:22:49,345 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_] to archive 2024-12-09T14:22:49,346 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T14:22:49,348 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ to hdfs://localhost:43873/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_ 2024-12-09T14:22:49,350 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1 to hdfs://localhost:43873/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/92c5578b19784bfa9fd1c032e533a6f1 2024-12-09T14:22:49,351 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ to hdfs://localhost:43873/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/ced8c13e42b547d2baa59be106051db7_SeqId_4_ 2024-12-09T14:22:49,353 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ to hdfs://localhost:43873/hbase/archive/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/a/fcbc5dde2de344759d16146303cd401a_SeqId_4_ 2024-12-09T14:22:49,353 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,353 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,354 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName b 2024-12-09T14:22:49,354 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,355 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,355 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,356 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee787dff136e27b2ef1ba5db3359e9df columnFamilyName c 2024-12-09T14:22:49,356 DEBUG [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,357 INFO [StoreOpener-ee787dff136e27b2ef1ba5db3359e9df-1 {}] regionserver.HStore(327): Store=ee787dff136e27b2ef1ba5db3359e9df/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,357 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,358 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,360 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,360 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008 2024-12-09T14:22:49,362 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:49,364 DEBUG [Time-limited test {}] regionserver.HRegion(5836): ee787dff136e27b2ef1ba5db3359e9df : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "ee787dff136e27b2ef1ba5db3359e9df" family_name: "a" compaction_input: "92c5578b19784bfa9fd1c032e533a6f1" compaction_input: "ced8c13e42b547d2baa59be106051db7_SeqId_4_" compaction_input: "83f875a1871b4b4fa82d1d11ce6ffaf1_SeqId_4_" compaction_input: "fcbc5dde2de344759d16146303cd401a_SeqId_4_" compaction_output: "854235b7133243f0961649381168a244" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T14:22:49,364 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T14:22:49,365 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008 2024-12-09T14:22:49,365 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/0000000000000000008 2024-12-09T14:22:49,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,367 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:49,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee787dff136e27b2ef1ba5db3359e9df 2024-12-09T14:22:49,372 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testCompactedBulkLoadedFiles/ee787dff136e27b2ef1ba5db3359e9df/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T14:22:49,373 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee787dff136e27b2ef1ba5db3359e9df; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72125845, jitterRate=0.07475884258747101}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:49,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee787dff136e27b2ef1ba5db3359e9df: Writing region info on filesystem at 1733754169313Initializing all the Stores at 1733754169315 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169315Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169315Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169315Cleaning up temporary data from old regions at 1733754169367 (+52 ms)Region opened successfully at 1733754169373 (+6 ms) 2024-12-09T14:22:49,376 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee787dff136e27b2ef1ba5db3359e9df, disabling compactions & flushes 2024-12-09T14:22:49,376 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:49,376 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:49,376 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. after waiting 0 ms 2024-12-09T14:22:49,376 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:49,377 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733754164455.ee787dff136e27b2ef1ba5db3359e9df. 2024-12-09T14:22:49,377 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee787dff136e27b2ef1ba5db3359e9df: Waiting for close lock at 1733754169376Disabling compacts and flushes for region at 1733754169376Disabling writes for close at 1733754169376Writing region close event to WAL at 1733754169377 (+1 ms)Closed at 1733754169377 2024-12-09T14:22:49,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741924_1103 (size=95) 2024-12-09T14:22:49,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741924_1103 (size=95) 2024-12-09T14:22:49,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741924_1103 (size=95) 2024-12-09T14:22:49,383 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:49,384 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733754169288) 2024-12-09T14:22:49,403 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=438 (was 428) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782268349_22 at /127.0.0.1:45048 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782268349_22 at /127.0.0.1:37740 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782268349_22 at /127.0.0.1:42354 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43873 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:43873 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1340 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=614 (was 572) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5486 (was 5490) 2024-12-09T14:22:49,403 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1340 is superior to 1024 2024-12-09T14:22:49,419 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1340, MaxFileDescriptor=1048576, SystemLoadAverage=614, ProcessCount=11, AvailableMemoryMB=5485 2024-12-09T14:22:49,419 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1340 is superior to 1024 2024-12-09T14:22:49,439 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:49,441 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:22:49,442 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:22:49,445 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82278861, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/hregion-82278861, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:49,462 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82278861/hregion-82278861.1733754169445, exclude list is [], retry=0 2024-12-09T14:22:49,465 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:49,465 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:49,466 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:49,468 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82278861/hregion-82278861.1733754169445 2024-12-09T14:22:49,468 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:49,468 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c1529614e0c8408b8e15a07884ee8915, NAME => 'testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43873/hbase 2024-12-09T14:22:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741926_1105 (size=67) 2024-12-09T14:22:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741926_1105 (size=67) 2024-12-09T14:22:49,480 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:49,482 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,484 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName a 2024-12-09T14:22:49,484 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741926_1105 (size=67) 2024-12-09T14:22:49,485 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,485 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,486 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName b 2024-12-09T14:22:49,486 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,487 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,487 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,488 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName c 2024-12-09T14:22:49,488 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,489 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,489 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,492 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,492 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,493 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:49,494 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,497 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:22:49,498 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1529614e0c8408b8e15a07884ee8915; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74811934, jitterRate=0.11478468775749207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:49,498 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1529614e0c8408b8e15a07884ee8915: Writing region info on filesystem at 1733754169480Initializing all the Stores at 1733754169482 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169482Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169482Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169482Cleaning up temporary data from old regions at 1733754169492 (+10 ms)Region opened successfully at 1733754169498 (+6 ms) 2024-12-09T14:22:49,498 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1529614e0c8408b8e15a07884ee8915, disabling compactions & flushes 2024-12-09T14:22:49,498 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,498 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,498 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. after waiting 0 ms 2024-12-09T14:22:49,498 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,499 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,499 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1529614e0c8408b8e15a07884ee8915: Waiting for close lock at 1733754169498Disabling compacts and flushes for region at 1733754169498Disabling writes for close at 1733754169498Writing region close event to WAL at 1733754169498Closed at 1733754169498 2024-12-09T14:22:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741925_1104 (size=95) 2024-12-09T14:22:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741925_1104 (size=95) 2024-12-09T14:22:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741925_1104 (size=95) 2024-12-09T14:22:49,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:49,510 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82278861:(num 1733754169445) 2024-12-09T14:22:49,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:49,512 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:49,531 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513, exclude list is [], retry=0 2024-12-09T14:22:49,534 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:49,535 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:49,535 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:49,537 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 2024-12-09T14:22:49,537 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:44507:44507),(127.0.0.1/127.0.0.1:33273:33273)] 2024-12-09T14:22:49,537 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1529614e0c8408b8e15a07884ee8915, NAME => 'testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:49,538 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:49,538 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,538 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,547 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,548 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName a 2024-12-09T14:22:49,549 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,549 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,549 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,550 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName b 2024-12-09T14:22:49,550 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,551 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,551 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,552 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName c 2024-12-09T14:22:49,552 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,553 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,553 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,556 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,556 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,557 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:49,559 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,561 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1529614e0c8408b8e15a07884ee8915; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60780938, jitterRate=-0.09429344534873962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:49,561 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1529614e0c8408b8e15a07884ee8915: Writing region info on filesystem at 1733754169538Initializing all the Stores at 1733754169539 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169539Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169547 (+8 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169547Cleaning up temporary data from old regions at 1733754169557 (+10 ms)Region opened successfully at 1733754169561 (+4 ms) 2024-12-09T14:22:49,572 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1529614e0c8408b8e15a07884ee8915 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T14:22:49,593 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/a/22e4d89406ad4e0abf8a2a8edf4b428a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733754169561/Put/seqid=0 2024-12-09T14:22:49,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741928_1107 (size=5958) 2024-12-09T14:22:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741928_1107 (size=5958) 2024-12-09T14:22:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741928_1107 (size=5958) 2024-12-09T14:22:49,610 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/a/22e4d89406ad4e0abf8a2a8edf4b428a 2024-12-09T14:22:49,621 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/a/22e4d89406ad4e0abf8a2a8edf4b428a as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/a/22e4d89406ad4e0abf8a2a8edf4b428a 2024-12-09T14:22:49,631 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/a/22e4d89406ad4e0abf8a2a8edf4b428a, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T14:22:49,633 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c1529614e0c8408b8e15a07884ee8915 in 61ms, sequenceid=13, compaction requested=false 2024-12-09T14:22:49,633 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c1529614e0c8408b8e15a07884ee8915: 2024-12-09T14:22:49,659 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1529614e0c8408b8e15a07884ee8915, disabling compactions & flushes 2024-12-09T14:22:49,660 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,660 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,660 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. after waiting 0 ms 2024-12-09T14:22:49,660 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,661 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,661 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:49,661 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1529614e0c8408b8e15a07884ee8915: Waiting for close lock at 1733754169659Disabling compacts and flushes for region at 1733754169659Disabling writes for close at 1733754169660 (+1 ms)Writing region close event to WAL at 1733754169661 (+1 ms)Closed at 1733754169661 2024-12-09T14:22:49,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741927_1106 (size=3345) 2024-12-09T14:22:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741927_1106 (size=3345) 2024-12-09T14:22:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741927_1106 (size=3345) 2024-12-09T14:22:49,691 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513, size=3.3 K (3345bytes) 2024-12-09T14:22:49,691 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 2024-12-09T14:22:49,692 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 after 1ms 2024-12-09T14:22:49,695 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:49,695 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 took 5ms 2024-12-09T14:22:49,697 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 so closing down 2024-12-09T14:22:49,698 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:49,699 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733754169513.temp 2024-12-09T14:22:49,701 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000003-wal.1733754169513.temp 2024-12-09T14:22:49,702 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:49,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741929_1108 (size=2944) 2024-12-09T14:22:49,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741929_1108 (size=2944) 2024-12-09T14:22:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741929_1108 (size=2944) 2024-12-09T14:22:49,715 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000003-wal.1733754169513.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:49,717 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000003-wal.1733754169513.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035 2024-12-09T14:22:49,717 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-12-09T14:22:49,718 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513, journal: Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513, size=3.3 K (3345bytes) at 1733754169691Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 so closing down at 1733754169698 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000003-wal.1733754169513.temp at 1733754169701 (+3 ms)3 split writer threads finished at 1733754169702 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000003-wal.1733754169513.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733754169715 (+13 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000003-wal.1733754169513.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035 at 1733754169717 (+2 ms)Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1733754169718 (+1 ms) 2024-12-09T14:22:49,720 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169513 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754169513 2024-12-09T14:22:49,721 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035 2024-12-09T14:22:49,722 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:49,724 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:49,743 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724, exclude list is [], retry=0 2024-12-09T14:22:49,746 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:49,746 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:49,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:49,753 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 2024-12-09T14:22:49,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:49,754 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1529614e0c8408b8e15a07884ee8915, NAME => 'testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:22:49,754 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:49,754 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,754 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,756 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,757 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName a 2024-12-09T14:22:49,757 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,764 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/a/22e4d89406ad4e0abf8a2a8edf4b428a 2024-12-09T14:22:49,764 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,764 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T14:22:49,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:49,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T14:22:49,765 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T14:22:49,767 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName b 2024-12-09T14:22:49,767 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,767 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,767 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,768 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName c 2024-12-09T14:22:49,768 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:49,769 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:49,769 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,772 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,774 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035 2024-12-09T14:22:49,778 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:49,779 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035 2024-12-09T14:22:49,779 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1529614e0c8408b8e15a07884ee8915 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T14:22:49,844 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/b/e03f705ad6ad4430b6df9478514965f4 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733754169633/Put/seqid=0 2024-12-09T14:22:49,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741931_1110 (size=5958) 2024-12-09T14:22:49,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741931_1110 (size=5958) 2024-12-09T14:22:49,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741931_1110 (size=5958) 2024-12-09T14:22:49,859 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/b/e03f705ad6ad4430b6df9478514965f4 2024-12-09T14:22:49,893 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/c/bf8f1f575996473bb8bad37bc3f8be0f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733754169646/Put/seqid=0 2024-12-09T14:22:49,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741932_1111 (size=5958) 2024-12-09T14:22:49,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741932_1111 (size=5958) 2024-12-09T14:22:49,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741932_1111 (size=5958) 2024-12-09T14:22:49,908 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/c/bf8f1f575996473bb8bad37bc3f8be0f 2024-12-09T14:22:49,917 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/b/e03f705ad6ad4430b6df9478514965f4 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/b/e03f705ad6ad4430b6df9478514965f4 2024-12-09T14:22:49,925 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/b/e03f705ad6ad4430b6df9478514965f4, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T14:22:49,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/c/bf8f1f575996473bb8bad37bc3f8be0f as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/c/bf8f1f575996473bb8bad37bc3f8be0f 2024-12-09T14:22:49,935 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/c/bf8f1f575996473bb8bad37bc3f8be0f, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T14:22:49,935 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c1529614e0c8408b8e15a07884ee8915 in 156ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T14:22:49,936 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000035 2024-12-09T14:22:49,938 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,938 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,939 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:49,940 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:49,945 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T14:22:49,946 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1529614e0c8408b8e15a07884ee8915; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70789706, jitterRate=0.054848819971084595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1529614e0c8408b8e15a07884ee8915: Writing region info on filesystem at 1733754169754Initializing all the Stores at 1733754169755 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169755Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169755Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754169755Obtaining lock to block concurrent updates at 1733754169779 (+24 ms)Preparing flush snapshotting stores in c1529614e0c8408b8e15a07884ee8915 at 1733754169779Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733754169779Flushing stores of testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. at 1733754169779Flushing c1529614e0c8408b8e15a07884ee8915/b: creating writer at 1733754169779Flushing c1529614e0c8408b8e15a07884ee8915/b: appending metadata at 1733754169843 (+64 ms)Flushing c1529614e0c8408b8e15a07884ee8915/b: closing flushed file at 1733754169843Flushing c1529614e0c8408b8e15a07884ee8915/c: creating writer at 1733754169868 (+25 ms)Flushing c1529614e0c8408b8e15a07884ee8915/c: appending metadata at 1733754169892 (+24 ms)Flushing c1529614e0c8408b8e15a07884ee8915/c: closing flushed file at 1733754169892Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b71554: reopening flushed file at 1733754169916 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b5a1064: reopening flushed file at 1733754169925 (+9 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c1529614e0c8408b8e15a07884ee8915 in 156ms, sequenceid=35, compaction requested=false; wal=null at 1733754169935 (+10 ms)Cleaning up temporary data from old regions at 1733754169938 (+3 ms)Region opened successfully at 1733754169946 (+8 ms) 2024-12-09T14:22:50,039 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724, size=0 (0bytes) 2024-12-09T14:22:50,039 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 might be still open, length is 0 2024-12-09T14:22:50,039 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 2024-12-09T14:22:50,040 WARN [IPC Server handler 2 on default port 43873 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T14:22:50,040 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 after 1ms 2024-12-09T14:22:52,851 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:45126 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45126 dst: /127.0.0.1:34769 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34769 remote=/127.0.0.1:45126]. Total timeout mills is 60000, 57135 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:52,851 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:42422 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34887:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42422 dst: /127.0.0.1:34887 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:52,851 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-171768464_22 at /127.0.0.1:37792 [Receiving block BP-728790427-172.17.0.3-1733754125568:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35501:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37792 dst: /127.0.0.1:35501 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:52,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741930_1112 (size=2936) 2024-12-09T14:22:52,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741930_1112 (size=2936) 2024-12-09T14:22:53,080 INFO [master/f4e784dc7cb5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T14:22:53,080 INFO [master/f4e784dc7cb5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T14:22:54,041 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 after 4002ms 2024-12-09T14:22:54,044 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:54,045 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 took 4006ms 2024-12-09T14:22:54,047 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724; continuing. 2024-12-09T14:22:54,047 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 so closing down 2024-12-09T14:22:54,047 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:22:54,049 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733754169724.temp 2024-12-09T14:22:54,051 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000037-wal.1733754169724.temp 2024-12-09T14:22:54,051 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:22:54,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741933_1113 (size=2944) 2024-12-09T14:22:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741933_1113 (size=2944) 2024-12-09T14:22:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741933_1113 (size=2944) 2024-12-09T14:22:54,075 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000037-wal.1733754169724.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T14:22:54,076 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000037-wal.1733754169724.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066 2024-12-09T14:22:54,076 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:22:54,077 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724, journal: Splitting hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724, size=0 (0bytes) at 1733754170039Finishing writing output for hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 so closing down at 1733754174047 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000037-wal.1733754169724.temp at 1733754174051 (+4 ms)3 split writer threads finished at 1733754174051Closed recovered edits writer path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000037-wal.1733754169724.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733754174075 (+24 ms)Rename recovered edits hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000037-wal.1733754169724.temp to hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066 at 1733754174076 (+1 ms)Processed 30 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724, size=0, length=0, corrupted=false, cancelled=false at 1733754174076 2024-12-09T14:22:54,079 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 to hdfs://localhost:43873/hbase/oldWALs/wal.1733754169724 2024-12-09T14:22:54,080 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066 2024-12-09T14:22:54,080 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:22:54,082 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43873/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438, archiveDir=hdfs://localhost:43873/hbase/oldWALs, maxLogs=32 2024-12-09T14:22:54,102 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754174083, exclude list is [], retry=0 2024-12-09T14:22:54,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34769,DS-85c3e7f4-b12e-48d9-bf88-61ea25765e83,DISK] 2024-12-09T14:22:54,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35501,DS-5da0b2d7-a79e-4338-832f-f553d838b0d6,DISK] 2024-12-09T14:22:54,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34887,DS-a53f40d1-818e-413a-b669-5a24c58c008c,DISK] 2024-12-09T14:22:54,123 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754174083 2024-12-09T14:22:54,123 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41675:41675),(127.0.0.1/127.0.0.1:33273:33273),(127.0.0.1/127.0.0.1:44507:44507)] 2024-12-09T14:22:54,124 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:22:54,128 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,129 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName a 2024-12-09T14:22:54,129 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:54,143 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/a/22e4d89406ad4e0abf8a2a8edf4b428a 2024-12-09T14:22:54,144 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:54,144 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,145 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName b 2024-12-09T14:22:54,145 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:54,151 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/b/e03f705ad6ad4430b6df9478514965f4 2024-12-09T14:22:54,151 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:54,151 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,152 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1529614e0c8408b8e15a07884ee8915 columnFamilyName c 2024-12-09T14:22:54,153 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:22:54,160 DEBUG [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/c/bf8f1f575996473bb8bad37bc3f8be0f 2024-12-09T14:22:54,160 INFO [StoreOpener-c1529614e0c8408b8e15a07884ee8915-1 {}] regionserver.HStore(327): Store=c1529614e0c8408b8e15a07884ee8915/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:22:54,161 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,161 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,164 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,165 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066 2024-12-09T14:22:54,167 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-09T14:22:54,173 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066 2024-12-09T14:22:54,173 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1529614e0c8408b8e15a07884ee8915 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T14:22:54,196 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/a/0090b74053f8401b8f7cdd07a28bbcc9 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733754169955/Put/seqid=0 2024-12-09T14:22:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741935_1115 (size=5958) 2024-12-09T14:22:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741935_1115 (size=5958) 2024-12-09T14:22:54,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741935_1115 (size=5958) 2024-12-09T14:22:54,220 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/a/0090b74053f8401b8f7cdd07a28bbcc9 2024-12-09T14:22:54,249 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/b/8d6f809e6f7149b18efd149a72238f3f is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733754169966/Put/seqid=0 2024-12-09T14:22:54,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741936_1116 (size=5958) 2024-12-09T14:22:54,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741936_1116 (size=5958) 2024-12-09T14:22:54,274 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/b/8d6f809e6f7149b18efd149a72238f3f 2024-12-09T14:22:54,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741936_1116 (size=5958) 2024-12-09T14:22:54,308 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/c/7ed92aeda7f14f42b7aad5bb3af829b5 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733754169975/Put/seqid=0 2024-12-09T14:22:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741937_1117 (size=5958) 2024-12-09T14:22:54,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741937_1117 (size=5958) 2024-12-09T14:22:54,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741937_1117 (size=5958) 2024-12-09T14:22:54,331 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/c/7ed92aeda7f14f42b7aad5bb3af829b5 2024-12-09T14:22:54,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/a/0090b74053f8401b8f7cdd07a28bbcc9 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/a/0090b74053f8401b8f7cdd07a28bbcc9 2024-12-09T14:22:54,344 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/a/0090b74053f8401b8f7cdd07a28bbcc9, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T14:22:54,346 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/b/8d6f809e6f7149b18efd149a72238f3f as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/b/8d6f809e6f7149b18efd149a72238f3f 2024-12-09T14:22:54,351 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/b/8d6f809e6f7149b18efd149a72238f3f, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T14:22:54,352 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/.tmp/c/7ed92aeda7f14f42b7aad5bb3af829b5 as hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/c/7ed92aeda7f14f42b7aad5bb3af829b5 2024-12-09T14:22:54,358 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/c/7ed92aeda7f14f42b7aad5bb3af829b5, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T14:22:54,358 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c1529614e0c8408b8e15a07884ee8915 in 185ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T14:22:54,359 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/0000000000000000066 2024-12-09T14:22:54,360 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,361 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,361 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:22:54,364 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1529614e0c8408b8e15a07884ee8915 2024-12-09T14:22:54,367 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/hbase/data/default/testReplayEditsWrittenViaHRegion/c1529614e0c8408b8e15a07884ee8915/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T14:22:54,368 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1529614e0c8408b8e15a07884ee8915; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61836178, jitterRate=-0.07856914401054382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:22:54,368 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1529614e0c8408b8e15a07884ee8915: Writing region info on filesystem at 1733754174124Initializing all the Stores at 1733754174127 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754174127Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754174127Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733754174127Obtaining lock to block concurrent updates at 1733754174173 (+46 ms)Preparing flush snapshotting stores in c1529614e0c8408b8e15a07884ee8915 at 1733754174173Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733754174173Flushing stores of testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. at 1733754174173Flushing c1529614e0c8408b8e15a07884ee8915/a: creating writer at 1733754174174 (+1 ms)Flushing c1529614e0c8408b8e15a07884ee8915/a: appending metadata at 1733754174196 (+22 ms)Flushing c1529614e0c8408b8e15a07884ee8915/a: closing flushed file at 1733754174196Flushing c1529614e0c8408b8e15a07884ee8915/b: creating writer at 1733754174227 (+31 ms)Flushing c1529614e0c8408b8e15a07884ee8915/b: appending metadata at 1733754174248 (+21 ms)Flushing c1529614e0c8408b8e15a07884ee8915/b: closing flushed file at 1733754174249 (+1 ms)Flushing c1529614e0c8408b8e15a07884ee8915/c: creating writer at 1733754174286 (+37 ms)Flushing c1529614e0c8408b8e15a07884ee8915/c: appending metadata at 1733754174307 (+21 ms)Flushing c1529614e0c8408b8e15a07884ee8915/c: closing flushed file at 1733754174308 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c9baa4f: reopening flushed file at 1733754174337 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@778339c3: reopening flushed file at 1733754174345 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15aa4282: reopening flushed file at 1733754174351 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c1529614e0c8408b8e15a07884ee8915 in 185ms, sequenceid=66, compaction requested=false; wal=null at 1733754174358 (+7 ms)Cleaning up temporary data from old regions at 1733754174361 (+3 ms)Region opened successfully at 1733754174368 (+7 ms) 2024-12-09T14:22:54,385 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1529614e0c8408b8e15a07884ee8915, disabling compactions & flushes 2024-12-09T14:22:54,385 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:54,385 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:54,385 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. after waiting 0 ms 2024-12-09T14:22:54,385 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:54,391 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733754169440.c1529614e0c8408b8e15a07884ee8915. 2024-12-09T14:22:54,391 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1529614e0c8408b8e15a07884ee8915: Waiting for close lock at 1733754174385Disabling compacts and flushes for region at 1733754174385Disabling writes for close at 1733754174385Writing region close event to WAL at 1733754174391 (+6 ms)Closed at 1733754174391 2024-12-09T14:22:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741934_1114 (size=95) 2024-12-09T14:22:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741934_1114 (size=95) 2024-12-09T14:22:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741934_1114 (size=95) 2024-12-09T14:22:54,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:22:54,400 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733754174083) 2024-12-09T14:22:54,421 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=443 (was 438) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:43873 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42893 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1517909453_22 at /127.0.0.1:45202 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1517909453_22 at /127.0.0.1:37816 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43873 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:42893 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1517909453_22 at /127.0.0.1:42452 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1404 (was 1340) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=621 (was 614) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5450 (was 5485) 2024-12-09T14:22:54,421 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1404 is superior to 1024 2024-12-09T14:22:54,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T14:22:54,422 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:22:54,422 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:22:54,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T14:22:54,423 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T14:22:54,423 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=223486682, stopped=false 2024-12-09T14:22:54,424 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f4e784dc7cb5,39231,1733754129274 2024-12-09T14:22:54,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:54,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:54,426 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:22:54,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:22:54,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:54,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:54,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:54,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:54,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:54,428 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:22:54,428 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:22:54,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,429 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f4e784dc7cb5,33819,1733754130225' ***** 2024-12-09T14:22:54,429 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:22:54,429 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f4e784dc7cb5,40373,1733754130392' ***** 2024-12-09T14:22:54,429 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:22:54,429 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:22:54,429 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:22:54,429 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:22:54,429 INFO [RS:2;f4e784dc7cb5:40373 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:22:54,429 INFO [RS:0;f4e784dc7cb5:33819 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:22:54,429 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:22:54,429 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:22:54,429 INFO [RS:2;f4e784dc7cb5:40373 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:22:54,429 INFO [RS:0;f4e784dc7cb5:33819 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:22:54,429 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(959): stopping server f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:54,429 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:22:54,430 INFO [RS:0;f4e784dc7cb5:33819 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f4e784dc7cb5:33819. 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(3091): Received CLOSE for 04cac2e3195e2ed23ca8b3c75b748c09 2024-12-09T14:22:54,430 DEBUG [RS:0;f4e784dc7cb5:33819 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:22:54,430 DEBUG [RS:0;f4e784dc7cb5:33819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,430 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(976): stopping server f4e784dc7cb5,33819,1733754130225; all regions closed. 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(959): stopping server f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f4e784dc7cb5:40373. 2024-12-09T14:22:54,430 DEBUG [RS:2;f4e784dc7cb5:40373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:22:54,430 DEBUG [RS:2;f4e784dc7cb5:40373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:22:54,430 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T14:22:54,431 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T14:22:54,431 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1325): Online Regions={04cac2e3195e2ed23ca8b3c75b748c09=testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09., 1588230740=hbase:meta,,1.1588230740} 2024-12-09T14:22:54,431 DEBUG [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1351): Waiting on 04cac2e3195e2ed23ca8b3c75b748c09, 1588230740 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 04cac2e3195e2ed23ca8b3c75b748c09, disabling compactions & flushes 2024-12-09T14:22:54,431 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. after waiting 0 ms 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:54,431 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:22:54,431 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:22:54,432 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T14:22:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741835_1011 (size=1708) 2024-12-09T14:22:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741835_1011 (size=1708) 2024-12-09T14:22:54,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741835_1011 (size=1708) 2024-12-09T14:22:54,456 DEBUG [RS:0;f4e784dc7cb5:33819 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs 2024-12-09T14:22:54,456 INFO [RS:0;f4e784dc7cb5:33819 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f4e784dc7cb5%2C33819%2C1733754130225:(num 1733754131925) 2024-12-09T14:22:54,456 DEBUG [RS:0;f4e784dc7cb5:33819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,456 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.ChoreService(370): Chore service for: regionserver/f4e784dc7cb5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:22:54,457 INFO [RS:0;f4e784dc7cb5:33819 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33819 2024-12-09T14:22:54,458 INFO [regionserver/f4e784dc7cb5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:22:54,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:22:54,460 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/04cac2e3195e2ed23ca8b3c75b748c09/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T14:22:54,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f4e784dc7cb5,33819,1733754130225 2024-12-09T14:22:54,460 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:22:54,461 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f4e784dc7cb5,33819,1733754130225] 2024-12-09T14:22:54,461 INFO [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:54,461 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 04cac2e3195e2ed23ca8b3c75b748c09: Waiting for close lock at 1733754174431Running coprocessor pre-close hooks at 1733754174431Disabling compacts and flushes for region at 1733754174431Disabling writes for close at 1733754174431Writing region close event to WAL at 1733754174453 (+22 ms)Running coprocessor post-close hooks at 1733754174461 (+8 ms)Closed at 1733754174461 2024-12-09T14:22:54,461 DEBUG [RS_CLOSE_REGION-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09. 2024-12-09T14:22:54,469 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f4e784dc7cb5,33819,1733754130225 already deleted, retry=false 2024-12-09T14:22:54,469 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f4e784dc7cb5,33819,1733754130225 expired; onlineServers=1 2024-12-09T14:22:54,483 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/info/b1d10f55fcb64335bd73a1bcb92e422f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733754148785.04cac2e3195e2ed23ca8b3c75b748c09./info:regioninfo/1733754152396/Put/seqid=0 2024-12-09T14:22:54,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741938_1118 (size=8243) 2024-12-09T14:22:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741938_1118 (size=8243) 2024-12-09T14:22:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741938_1118 (size=8243) 2024-12-09T14:22:54,494 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/info/b1d10f55fcb64335bd73a1bcb92e422f 2024-12-09T14:22:54,521 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/ns/3bae55b7f4334f5086f109ac972a20ae is 43, key is default/ns:d/1733754132506/Put/seqid=0 2024-12-09T14:22:54,526 INFO [regionserver/f4e784dc7cb5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:22:54,526 INFO [regionserver/f4e784dc7cb5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:22:54,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741939_1119 (size=5153) 2024-12-09T14:22:54,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741939_1119 (size=5153) 2024-12-09T14:22:54,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741939_1119 (size=5153) 2024-12-09T14:22:54,539 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/ns/3bae55b7f4334f5086f109ac972a20ae 2024-12-09T14:22:54,563 INFO [RS:0;f4e784dc7cb5:33819 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:22:54,563 INFO [RS:0;f4e784dc7cb5:33819 {}] regionserver.HRegionServer(1031): Exiting; stopping=f4e784dc7cb5,33819,1733754130225; zookeeper connection closed. 2024-12-09T14:22:54,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:54,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33819-0x1012b90185b0001, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:54,563 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@182ea00b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@182ea00b 2024-12-09T14:22:54,567 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/table/43f2b760c5ca4881b0469bf7801a3a9a is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733754149303/Put/seqid=0 2024-12-09T14:22:54,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741940_1120 (size=5431) 2024-12-09T14:22:54,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741940_1120 (size=5431) 2024-12-09T14:22:54,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741940_1120 (size=5431) 2024-12-09T14:22:54,583 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/table/43f2b760c5ca4881b0469bf7801a3a9a 2024-12-09T14:22:54,591 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/info/b1d10f55fcb64335bd73a1bcb92e422f as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/info/b1d10f55fcb64335bd73a1bcb92e422f 2024-12-09T14:22:54,598 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/info/b1d10f55fcb64335bd73a1bcb92e422f, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T14:22:54,600 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/ns/3bae55b7f4334f5086f109ac972a20ae as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/ns/3bae55b7f4334f5086f109ac972a20ae 2024-12-09T14:22:54,606 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/ns/3bae55b7f4334f5086f109ac972a20ae, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T14:22:54,608 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/.tmp/table/43f2b760c5ca4881b0469bf7801a3a9a as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/table/43f2b760c5ca4881b0469bf7801a3a9a 2024-12-09T14:22:54,615 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/table/43f2b760c5ca4881b0469bf7801a3a9a, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T14:22:54,617 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 186ms, sequenceid=21, compaction requested=false 2024-12-09T14:22:54,629 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T14:22:54,630 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:22:54,631 INFO [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:22:54,631 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733754174431Running coprocessor pre-close hooks at 1733754174431Disabling compacts and flushes for region at 1733754174431Disabling writes for close at 1733754174431Obtaining lock to block concurrent updates at 1733754174432 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733754174432Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733754174432Flushing stores of hbase:meta,,1.1588230740 at 1733754174433 (+1 ms)Flushing 1588230740/info: creating writer at 1733754174433Flushing 1588230740/info: appending metadata at 1733754174482 (+49 ms)Flushing 1588230740/info: closing flushed file at 1733754174482Flushing 1588230740/ns: creating writer at 1733754174500 (+18 ms)Flushing 1588230740/ns: appending metadata at 1733754174520 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733754174520Flushing 1588230740/table: creating writer at 1733754174546 (+26 ms)Flushing 1588230740/table: appending metadata at 1733754174566 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733754174566Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f39e341: reopening flushed file at 1733754174590 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27b40c6b: reopening flushed file at 1733754174599 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@251f0429: reopening flushed file at 1733754174607 (+8 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 186ms, sequenceid=21, compaction requested=false at 1733754174617 (+10 ms)Writing region close event to WAL at 1733754174624 (+7 ms)Running coprocessor post-close hooks at 1733754174630 (+6 ms)Closed at 1733754174630 2024-12-09T14:22:54,631 DEBUG [RS_CLOSE_META-regionserver/f4e784dc7cb5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T14:22:54,631 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(976): stopping server f4e784dc7cb5,40373,1733754130392; all regions closed. 2024-12-09T14:22:54,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741836_1012 (size=4716) 2024-12-09T14:22:54,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741836_1012 (size=4716) 2024-12-09T14:22:54,644 DEBUG [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs 2024-12-09T14:22:54,644 INFO [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f4e784dc7cb5%2C40373%2C1733754130392.meta:.meta(num 1733754132315) 2024-12-09T14:22:54,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741833_1009 (size=721) 2024-12-09T14:22:54,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741833_1009 (size=721) 2024-12-09T14:22:54,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741833_1009 (size=721) 2024-12-09T14:22:54,652 DEBUG [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/oldWALs 2024-12-09T14:22:54,652 INFO [RS:2;f4e784dc7cb5:40373 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f4e784dc7cb5%2C40373%2C1733754130392:(num 1733754131916) 2024-12-09T14:22:54,652 DEBUG [RS:2;f4e784dc7cb5:40373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:22:54,653 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:22:54,653 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:22:54,653 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.ChoreService(370): Chore service for: regionserver/f4e784dc7cb5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:22:54,653 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:22:54,653 INFO [regionserver/f4e784dc7cb5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:22:54,653 INFO [RS:2;f4e784dc7cb5:40373 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40373 2024-12-09T14:22:54,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f4e784dc7cb5,40373,1733754130392 2024-12-09T14:22:54,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:22:54,656 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:22:54,658 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f4e784dc7cb5,40373,1733754130392] 2024-12-09T14:22:54,659 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f4e784dc7cb5,40373,1733754130392 already deleted, retry=false 2024-12-09T14:22:54,659 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f4e784dc7cb5,40373,1733754130392 expired; onlineServers=0 2024-12-09T14:22:54,660 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f4e784dc7cb5,39231,1733754129274' ***** 2024-12-09T14:22:54,660 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T14:22:54,660 INFO [M:0;f4e784dc7cb5:39231 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:22:54,660 INFO [M:0;f4e784dc7cb5:39231 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:22:54,660 DEBUG [M:0;f4e784dc7cb5:39231 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T14:22:54,660 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T14:22:54,660 DEBUG [M:0;f4e784dc7cb5:39231 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T14:22:54,660 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster-HFileCleaner.large.0-1733754131521 {}] cleaner.HFileCleaner(306): Exit Thread[master/f4e784dc7cb5:0:becomeActiveMaster-HFileCleaner.large.0-1733754131521,5,FailOnTimeoutGroup] 2024-12-09T14:22:54,661 INFO [M:0;f4e784dc7cb5:39231 {}] hbase.ChoreService(370): Chore service for: master/f4e784dc7cb5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T14:22:54,661 INFO [M:0;f4e784dc7cb5:39231 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:22:54,661 DEBUG [M:0;f4e784dc7cb5:39231 {}] master.HMaster(1795): Stopping service threads 2024-12-09T14:22:54,661 INFO [M:0;f4e784dc7cb5:39231 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T14:22:54,661 DEBUG [master/f4e784dc7cb5:0:becomeActiveMaster-HFileCleaner.small.0-1733754131527 {}] cleaner.HFileCleaner(306): Exit Thread[master/f4e784dc7cb5:0:becomeActiveMaster-HFileCleaner.small.0-1733754131527,5,FailOnTimeoutGroup] 2024-12-09T14:22:54,661 INFO [M:0;f4e784dc7cb5:39231 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:22:54,662 INFO [M:0;f4e784dc7cb5:39231 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T14:22:54,662 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T14:22:54,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T14:22:54,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:22:54,664 DEBUG [M:0;f4e784dc7cb5:39231 {}] zookeeper.ZKUtil(347): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T14:22:54,664 WARN [M:0;f4e784dc7cb5:39231 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T14:22:54,665 INFO [M:0;f4e784dc7cb5:39231 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/.lastflushedseqids 2024-12-09T14:22:54,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741941_1121 (size=138) 2024-12-09T14:22:54,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741941_1121 (size=138) 2024-12-09T14:22:54,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741941_1121 (size=138) 2024-12-09T14:22:54,687 INFO [M:0;f4e784dc7cb5:39231 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T14:22:54,687 INFO [M:0;f4e784dc7cb5:39231 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T14:22:54,687 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:22:54,687 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:54,687 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:54,687 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:22:54,687 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:54,687 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-09T14:22:54,708 DEBUG [M:0;f4e784dc7cb5:39231 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b9adfcf697d449658004ecea86be99bd is 82, key is hbase:meta,,1/info:regioninfo/1733754132416/Put/seqid=0 2024-12-09T14:22:54,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741942_1122 (size=5672) 2024-12-09T14:22:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741942_1122 (size=5672) 2024-12-09T14:22:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741942_1122 (size=5672) 2024-12-09T14:22:54,717 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b9adfcf697d449658004ecea86be99bd 2024-12-09T14:22:54,743 DEBUG [M:0;f4e784dc7cb5:39231 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ffaec37a96d4b199edbc9f324040ab5 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733754149314/Put/seqid=0 2024-12-09T14:22:54,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741943_1123 (size=7755) 2024-12-09T14:22:54,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:54,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40373-0x1012b90185b0003, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:54,758 INFO [RS:2;f4e784dc7cb5:40373 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:22:54,758 INFO [RS:2;f4e784dc7cb5:40373 {}] regionserver.HRegionServer(1031): Exiting; stopping=f4e784dc7cb5,40373,1733754130392; zookeeper connection closed. 2024-12-09T14:22:54,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741943_1123 (size=7755) 2024-12-09T14:22:54,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741943_1123 (size=7755) 2024-12-09T14:22:54,759 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3bde5e9f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3bde5e9f 2024-12-09T14:22:54,760 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ffaec37a96d4b199edbc9f324040ab5 2024-12-09T14:22:54,761 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T14:22:54,767 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1ffaec37a96d4b199edbc9f324040ab5 2024-12-09T14:22:54,789 DEBUG [M:0;f4e784dc7cb5:39231 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/31ef8ef8d36a48d9971e3d69ac64b21f is 69, key is f4e784dc7cb5,33819,1733754130225/rs:state/1733754131629/Put/seqid=0 2024-12-09T14:22:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741944_1124 (size=5440) 2024-12-09T14:22:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741944_1124 (size=5440) 2024-12-09T14:22:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741944_1124 (size=5440) 2024-12-09T14:22:54,810 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/31ef8ef8d36a48d9971e3d69ac64b21f 2024-12-09T14:22:54,816 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31ef8ef8d36a48d9971e3d69ac64b21f 2024-12-09T14:22:54,817 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b9adfcf697d449658004ecea86be99bd as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b9adfcf697d449658004ecea86be99bd 2024-12-09T14:22:54,823 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b9adfcf697d449658004ecea86be99bd, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T14:22:54,824 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ffaec37a96d4b199edbc9f324040ab5 as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1ffaec37a96d4b199edbc9f324040ab5 2024-12-09T14:22:54,831 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1ffaec37a96d4b199edbc9f324040ab5 2024-12-09T14:22:54,831 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1ffaec37a96d4b199edbc9f324040ab5, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T14:22:54,832 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/31ef8ef8d36a48d9971e3d69ac64b21f as hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/31ef8ef8d36a48d9971e3d69ac64b21f 2024-12-09T14:22:54,839 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31ef8ef8d36a48d9971e3d69ac64b21f 2024-12-09T14:22:54,839 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43873/user/jenkins/test-data/4af78ae7-ed06-1d15-6d2a-e03ea5bb7bf9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/31ef8ef8d36a48d9971e3d69ac64b21f, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T14:22:54,840 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=168, compaction requested=false 2024-12-09T14:22:54,847 INFO [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:22:54,847 DEBUG [M:0;f4e784dc7cb5:39231 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733754174687Disabling compacts and flushes for region at 1733754174687Disabling writes for close at 1733754174687Obtaining lock to block concurrent updates at 1733754174687Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733754174687Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733754174688 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733754174689 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733754174689Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733754174708 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733754174708Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733754174723 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733754174743 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733754174743Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733754174767 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733754174788 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733754174788Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45dd0eb8: reopening flushed file at 1733754174816 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26330485: reopening flushed file at 1733754174823 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2790fe6f: reopening flushed file at 1733754174831 (+8 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=168, compaction requested=false at 1733754174840 (+9 ms)Writing region close event to WAL at 1733754174847 (+7 ms)Closed at 1733754174847 2024-12-09T14:22:54,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35501 is added to blk_1073741830_1006 (size=56546) 2024-12-09T14:22:54,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34887 is added to blk_1073741830_1006 (size=56546) 2024-12-09T14:22:54,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34769 is added to blk_1073741830_1006 (size=56546) 2024-12-09T14:22:54,852 INFO [M:0;f4e784dc7cb5:39231 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T14:22:54,852 INFO [M:0;f4e784dc7cb5:39231 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39231 2024-12-09T14:22:54,852 INFO [M:0;f4e784dc7cb5:39231 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:22:54,852 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:22:54,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:54,954 INFO [M:0;f4e784dc7cb5:39231 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:22:54,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39231-0x1012b90185b0000, quorum=127.0.0.1:49681, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:22:54,962 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754153225 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754153225 (inode 16655) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733754152713/wal.1733754153225 (inode 16655) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T14:22:54,963 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733754144285/wal.1733754144413 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:54,963 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733754169438/wal.1733754169724 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:54,966 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754162823 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754162823 (inode 16678) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733754153446/wal.1733754162823 (inode 16678) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T14:22:54,966 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733754134199/wal.1733754134283 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:54,969 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163812 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163812 (inode 16704) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733754163097/wal.1733754163812 (inode 16704) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T14:22:54,969 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733754164450/wal.1733754165002 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:22:54,972 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal.1733754134040 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal.1733754134040 (inode 16485) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733754133315/wal.1733754134040 (inode 16485) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T14:22:54,974 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal.1733754133044 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal.1733754133044 (inode 16462) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733754132784/wal.1733754133044 (inode 16462) Holder DFSClient_NONMAPREDUCE_-171768464_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T14:22:55,005 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26f8cb79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:22:55,008 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:22:55,008 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:22:55,008 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:22:55,008 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,STOPPED} 2024-12-09T14:22:55,013 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T14:22:55,013 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T14:22:55,013 WARN [BP-728790427-172.17.0.3-1733754125568 heartbeating to localhost/127.0.0.1:43873 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T14:22:55,013 WARN [BP-728790427-172.17.0.3-1733754125568 heartbeating to localhost/127.0.0.1:43873 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-728790427-172.17.0.3-1733754125568 (Datanode Uuid 5db1e07e-9907-45e2-931e-a86e2b84132d) service to localhost/127.0.0.1:43873 2024-12-09T14:22:55,015 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data5/current/BP-728790427-172.17.0.3-1733754125568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:22:55,015 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data6/current/BP-728790427-172.17.0.3-1733754125568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:22:55,016 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T14:22:55,028 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4464725f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:22:55,028 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:22:55,028 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:22:55,028 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:22:55,029 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,STOPPED} 2024-12-09T14:22:55,032 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T14:22:55,032 WARN [BP-728790427-172.17.0.3-1733754125568 heartbeating to localhost/127.0.0.1:43873 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T14:22:55,032 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T14:22:55,032 WARN [BP-728790427-172.17.0.3-1733754125568 heartbeating to localhost/127.0.0.1:43873 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-728790427-172.17.0.3-1733754125568 (Datanode Uuid 0ae23200-61d1-484e-8543-bc293776a38b) service to localhost/127.0.0.1:43873 2024-12-09T14:22:55,033 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data3/current/BP-728790427-172.17.0.3-1733754125568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:22:55,033 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data4/current/BP-728790427-172.17.0.3-1733754125568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:22:55,034 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T14:22:55,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17d1a8d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:22:55,065 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:22:55,065 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:22:55,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:22:55,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,STOPPED} 2024-12-09T14:22:55,068 WARN [BP-728790427-172.17.0.3-1733754125568 heartbeating to localhost/127.0.0.1:43873 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T14:22:55,068 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T14:22:55,068 WARN [BP-728790427-172.17.0.3-1733754125568 heartbeating to localhost/127.0.0.1:43873 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-728790427-172.17.0.3-1733754125568 (Datanode Uuid d475c1ca-f353-47c5-aa02-45e4cda55273) service to localhost/127.0.0.1:43873 2024-12-09T14:22:55,068 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T14:22:55,069 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data1/current/BP-728790427-172.17.0.3-1733754125568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:22:55,069 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T14:22:55,069 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/cluster_d86f78c6-bdf5-5198-944b-1df9811e5c3d/data/data2/current/BP-728790427-172.17.0.3-1733754125568 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:22:55,079 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c8b857b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T14:22:55,080 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:22:55,080 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:22:55,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:22:55,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4b2281a-d611-a10f-816b-126ebc96f530/hadoop.log.dir/,STOPPED} 2024-12-09T14:22:55,094 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T14:22:55,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down