2024-11-14 21:49:59,191 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-14 21:49:59,206 main DEBUG Took 0.012410 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 21:49:59,206 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 21:49:59,207 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 21:49:59,208 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 21:49:59,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,218 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 21:49:59,229 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,230 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,231 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,231 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,231 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,232 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,233 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,234 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,235 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,235 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,236 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,236 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,236 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,237 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,238 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 21:49:59,238 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,238 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 21:49:59,240 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 21:49:59,241 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 21:49:59,242 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 21:49:59,243 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 21:49:59,244 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 21:49:59,244 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 21:49:59,252 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 21:49:59,254 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 21:49:59,256 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 21:49:59,256 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 21:49:59,256 main DEBUG createAppenders(={Console}) 2024-11-14 21:49:59,257 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-14 21:49:59,257 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-14 21:49:59,257 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-14 21:49:59,258 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 21:49:59,258 main DEBUG OutputStream closed 2024-11-14 21:49:59,258 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 21:49:59,259 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 21:49:59,259 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-14 21:49:59,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 21:49:59,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 21:49:59,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 21:49:59,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 21:49:59,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 21:49:59,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 21:49:59,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 21:49:59,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 21:49:59,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 21:49:59,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 21:49:59,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 21:49:59,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 21:49:59,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 21:49:59,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 21:49:59,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 21:49:59,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 21:49:59,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 21:49:59,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 21:49:59,341 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 21:49:59,342 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-14 21:49:59,342 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 21:49:59,343 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-14T21:49:59,615 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3 2024-11-14 21:49:59,618 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 21:49:59,618 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T21:49:59,628 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-14T21:49:59,658 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T21:49:59,704 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T21:49:59,704 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T21:49:59,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T21:49:59,732 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06, deleteOnExit=true 2024-11-14T21:49:59,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T21:49:59,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/test.cache.data in system properties and HBase conf 2024-11-14T21:49:59,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T21:49:59,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir in system properties and HBase conf 2024-11-14T21:49:59,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T21:49:59,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T21:49:59,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T21:49:59,823 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T21:49:59,917 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T21:49:59,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T21:49:59,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T21:49:59,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T21:49:59,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T21:49:59,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T21:49:59,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T21:49:59,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T21:49:59,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T21:49:59,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T21:49:59,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/nfs.dump.dir in system properties and HBase conf 2024-11-14T21:49:59,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/java.io.tmpdir in system properties and HBase conf 2024-11-14T21:49:59,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T21:49:59,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T21:49:59,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T21:50:00,762 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T21:50:00,844 INFO [Time-limited test {}] log.Log(170): Logging initialized @2346ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T21:50:00,926 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:50:00,989 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T21:50:01,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T21:50:01,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T21:50:01,012 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T21:50:01,028 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:50:01,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,AVAILABLE} 2024-11-14T21:50:01,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T21:50:01,261 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/java.io.tmpdir/jetty-localhost-41433-hadoop-hdfs-3_4_1-tests_jar-_-any-17028377072627167293/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T21:50:01,268 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:41433} 2024-11-14T21:50:01,268 INFO [Time-limited test {}] server.Server(415): Started @2771ms 2024-11-14T21:50:01,746 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:50:01,755 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T21:50:01,757 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T21:50:01,757 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T21:50:01,757 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T21:50:01,759 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,AVAILABLE} 2024-11-14T21:50:01,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T21:50:01,904 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/java.io.tmpdir/jetty-localhost-37919-hadoop-hdfs-3_4_1-tests_jar-_-any-15507277051930795319/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:50:01,905 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:37919} 2024-11-14T21:50:01,905 INFO [Time-limited test {}] server.Server(415): Started @3408ms 2024-11-14T21:50:01,964 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T21:50:02,120 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:50:02,127 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T21:50:02,134 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T21:50:02,134 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T21:50:02,134 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T21:50:02,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,AVAILABLE} 2024-11-14T21:50:02,136 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T21:50:02,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/java.io.tmpdir/jetty-localhost-38789-hadoop-hdfs-3_4_1-tests_jar-_-any-1324739654170772411/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:50:02,285 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:38789} 2024-11-14T21:50:02,285 INFO [Time-limited test {}] server.Server(415): Started @3787ms 2024-11-14T21:50:02,287 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T21:50:02,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T21:50:02,335 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T21:50:02,343 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T21:50:02,344 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T21:50:02,344 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T21:50:02,345 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,AVAILABLE} 2024-11-14T21:50:02,345 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T21:50:02,478 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/java.io.tmpdir/jetty-localhost-40961-hadoop-hdfs-3_4_1-tests_jar-_-any-14645223056763484318/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:50:02,479 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:40961} 2024-11-14T21:50:02,479 INFO [Time-limited test {}] server.Server(415): Started @3982ms 2024-11-14T21:50:02,481 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T21:50:02,491 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data3/current/BP-2030617010-172.17.0.2-1731621000517/current, will proceed with Du for space computation calculation, 2024-11-14T21:50:02,491 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data1/current/BP-2030617010-172.17.0.2-1731621000517/current, will proceed with Du for space computation calculation, 2024-11-14T21:50:02,491 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data4/current/BP-2030617010-172.17.0.2-1731621000517/current, will proceed with Du for space computation calculation, 2024-11-14T21:50:02,491 WARN [Thread-111 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data2/current/BP-2030617010-172.17.0.2-1731621000517/current, will proceed with Du for space computation calculation, 2024-11-14T21:50:02,535 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T21:50:02,535 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T21:50:02,613 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x923cc978fb8f57fe with lease ID 0xc0cde3522d1bccb8: Processing first storage report for DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0 from datanode DatanodeRegistration(127.0.0.1:32913, datanodeUuid=1a2405d8-c26c-47fc-9586-bd1c92159f32, infoPort=34769, infoSecurePort=0, ipcPort=44607, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517) 2024-11-14T21:50:02,615 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x923cc978fb8f57fe with lease ID 0xc0cde3522d1bccb8: from storage DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0 node DatanodeRegistration(127.0.0.1:32913, datanodeUuid=1a2405d8-c26c-47fc-9586-bd1c92159f32, infoPort=34769, infoSecurePort=0, ipcPort=44607, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T21:50:02,615 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca5555721932b21b with lease ID 0xc0cde3522d1bccb9: Processing first storage report for DS-cc2247b8-7b78-4297-b18c-eecee0d0563e from datanode DatanodeRegistration(127.0.0.1:39681, datanodeUuid=7fe6d6ea-3168-4cc0-aaf1-cdfb65011532, infoPort=43637, infoSecurePort=0, ipcPort=33265, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517) 2024-11-14T21:50:02,616 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca5555721932b21b with lease ID 0xc0cde3522d1bccb9: from storage DS-cc2247b8-7b78-4297-b18c-eecee0d0563e node DatanodeRegistration(127.0.0.1:39681, datanodeUuid=7fe6d6ea-3168-4cc0-aaf1-cdfb65011532, infoPort=43637, infoSecurePort=0, ipcPort=33265, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T21:50:02,616 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x923cc978fb8f57fe with lease ID 0xc0cde3522d1bccb8: Processing first storage report for DS-e7d07b14-fba5-4b15-b40d-aa394578afeb from datanode DatanodeRegistration(127.0.0.1:32913, datanodeUuid=1a2405d8-c26c-47fc-9586-bd1c92159f32, infoPort=34769, infoSecurePort=0, ipcPort=44607, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517) 2024-11-14T21:50:02,616 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x923cc978fb8f57fe with lease ID 0xc0cde3522d1bccb8: from storage DS-e7d07b14-fba5-4b15-b40d-aa394578afeb node DatanodeRegistration(127.0.0.1:32913, datanodeUuid=1a2405d8-c26c-47fc-9586-bd1c92159f32, infoPort=34769, infoSecurePort=0, ipcPort=44607, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T21:50:02,616 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca5555721932b21b with lease ID 0xc0cde3522d1bccb9: Processing first storage report for DS-418e9930-3859-4aae-9e7b-80c0a56243e7 from datanode DatanodeRegistration(127.0.0.1:39681, datanodeUuid=7fe6d6ea-3168-4cc0-aaf1-cdfb65011532, infoPort=43637, infoSecurePort=0, ipcPort=33265, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517) 2024-11-14T21:50:02,617 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca5555721932b21b with lease ID 0xc0cde3522d1bccb9: from storage DS-418e9930-3859-4aae-9e7b-80c0a56243e7 node DatanodeRegistration(127.0.0.1:39681, datanodeUuid=7fe6d6ea-3168-4cc0-aaf1-cdfb65011532, infoPort=43637, infoSecurePort=0, ipcPort=33265, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T21:50:02,631 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data5/current/BP-2030617010-172.17.0.2-1731621000517/current, will proceed with Du for space computation calculation, 2024-11-14T21:50:02,633 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data6/current/BP-2030617010-172.17.0.2-1731621000517/current, will proceed with Du for space computation calculation, 2024-11-14T21:50:02,663 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T21:50:02,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa54fba00aca017b with lease ID 0xc0cde3522d1bccba: Processing first storage report for DS-a651bf80-bdd1-4033-86ca-671a3ec818a6 from datanode DatanodeRegistration(127.0.0.1:44645, datanodeUuid=80cbac55-7fc0-4e95-99ef-4db521160716, infoPort=40169, infoSecurePort=0, ipcPort=44719, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517) 2024-11-14T21:50:02,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa54fba00aca017b with lease ID 0xc0cde3522d1bccba: from storage DS-a651bf80-bdd1-4033-86ca-671a3ec818a6 node DatanodeRegistration(127.0.0.1:44645, datanodeUuid=80cbac55-7fc0-4e95-99ef-4db521160716, infoPort=40169, infoSecurePort=0, ipcPort=44719, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T21:50:02,670 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa54fba00aca017b with lease ID 0xc0cde3522d1bccba: Processing first storage report for DS-ba22ec8f-5db5-4085-a113-73128adc1dd7 from datanode DatanodeRegistration(127.0.0.1:44645, datanodeUuid=80cbac55-7fc0-4e95-99ef-4db521160716, infoPort=40169, infoSecurePort=0, ipcPort=44719, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517) 2024-11-14T21:50:02,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa54fba00aca017b with lease ID 0xc0cde3522d1bccba: from storage DS-ba22ec8f-5db5-4085-a113-73128adc1dd7 node DatanodeRegistration(127.0.0.1:44645, datanodeUuid=80cbac55-7fc0-4e95-99ef-4db521160716, infoPort=40169, infoSecurePort=0, ipcPort=44719, storageInfo=lv=-57;cid=testClusterID;nsid=1016059626;c=1731621000517), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T21:50:02,900 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3 2024-11-14T21:50:02,975 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/zookeeper_0, clientPort=60020, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T21:50:02,985 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60020 2024-11-14T21:50:02,997 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:03,001 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:03,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741825_1001 (size=7) 2024-11-14T21:50:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741825_1001 (size=7) 2024-11-14T21:50:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741825_1001 (size=7) 2024-11-14T21:50:03,648 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6 with version=8 2024-11-14T21:50:03,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/hbase-staging 2024-11-14T21:50:03,994 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef788a4cade8:0 server-side Connection retries=45 2024-11-14T21:50:04,005 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,006 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,011 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T21:50:04,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T21:50:04,192 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T21:50:04,269 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T21:50:04,279 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T21:50:04,284 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T21:50:04,318 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24913 (auto-detected) 2024-11-14T21:50:04,319 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T21:50:04,346 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37243 2024-11-14T21:50:04,372 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37243 connecting to ZooKeeper ensemble=127.0.0.1:60020 2024-11-14T21:50:04,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:372430x0, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T21:50:04,420 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37243-0x100ac6b2eed0000 connected 2024-11-14T21:50:04,459 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:04,480 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6, hbase.cluster.distributed=false 2024-11-14T21:50:04,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T21:50:04,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37243 2024-11-14T21:50:04,526 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37243 2024-11-14T21:50:04,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37243 2024-11-14T21:50:04,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37243 2024-11-14T21:50:04,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37243 2024-11-14T21:50:04,667 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef788a4cade8:0 server-side Connection retries=45 2024-11-14T21:50:04,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,669 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T21:50:04,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T21:50:04,673 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T21:50:04,675 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T21:50:04,676 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39395 2024-11-14T21:50:04,679 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39395 connecting to ZooKeeper ensemble=127.0.0.1:60020 2024-11-14T21:50:04,680 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,682 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393950x0, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T21:50:04,689 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39395-0x100ac6b2eed0001 connected 2024-11-14T21:50:04,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:04,694 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T21:50:04,702 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T21:50:04,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T21:50:04,709 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T21:50:04,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39395 2024-11-14T21:50:04,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39395 2024-11-14T21:50:04,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39395 2024-11-14T21:50:04,711 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39395 2024-11-14T21:50:04,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39395 2024-11-14T21:50:04,730 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef788a4cade8:0 server-side Connection retries=45 2024-11-14T21:50:04,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,731 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T21:50:04,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,732 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T21:50:04,732 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T21:50:04,732 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T21:50:04,733 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36817 2024-11-14T21:50:04,735 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36817 connecting to ZooKeeper ensemble=127.0.0.1:60020 2024-11-14T21:50:04,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,740 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368170x0, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T21:50:04,747 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36817-0x100ac6b2eed0002 connected 2024-11-14T21:50:04,748 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:04,749 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T21:50:04,751 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T21:50:04,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T21:50:04,755 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T21:50:04,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36817 2024-11-14T21:50:04,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36817 2024-11-14T21:50:04,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36817 2024-11-14T21:50:04,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36817 2024-11-14T21:50:04,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36817 2024-11-14T21:50:04,776 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef788a4cade8:0 server-side Connection retries=45 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T21:50:04,777 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T21:50:04,779 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36167 2024-11-14T21:50:04,780 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36167 connecting to ZooKeeper ensemble=127.0.0.1:60020 2024-11-14T21:50:04,781 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,784 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361670x0, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T21:50:04,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361670x0, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:04,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36167-0x100ac6b2eed0003 connected 2024-11-14T21:50:04,791 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T21:50:04,793 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T21:50:04,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T21:50:04,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T21:50:04,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36167 2024-11-14T21:50:04,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36167 2024-11-14T21:50:04,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36167 2024-11-14T21:50:04,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36167 2024-11-14T21:50:04,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36167 2024-11-14T21:50:04,817 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef788a4cade8:37243 2024-11-14T21:50:04,819 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef788a4cade8,37243,1731621003759 2024-11-14T21:50:04,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,830 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef788a4cade8,37243,1731621003759 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:04,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:04,855 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T21:50:04,857 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef788a4cade8,37243,1731621003759 from backup master directory 2024-11-14T21:50:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef788a4cade8,37243,1731621003759 2024-11-14T21:50:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T21:50:04,861 WARN [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T21:50:04,861 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef788a4cade8,37243,1731621003759 2024-11-14T21:50:04,863 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T21:50:04,865 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T21:50:04,923 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/hbase.id] with ID: 5db27ac6-dee0-4c18-b507-d6be882807e2 2024-11-14T21:50:04,923 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/.tmp/hbase.id 2024-11-14T21:50:04,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741826_1002 (size=42) 2024-11-14T21:50:04,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741826_1002 (size=42) 2024-11-14T21:50:04,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741826_1002 (size=42) 2024-11-14T21:50:04,939 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/.tmp/hbase.id]:[hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/hbase.id] 2024-11-14T21:50:04,985 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T21:50:04,990 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T21:50:05,011 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-14T21:50:05,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741827_1003 (size=196) 2024-11-14T21:50:05,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741827_1003 (size=196) 2024-11-14T21:50:05,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741827_1003 (size=196) 2024-11-14T21:50:05,051 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:50:05,053 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T21:50:05,059 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:05,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741828_1004 (size=1189) 2024-11-14T21:50:05,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741828_1004 (size=1189) 2024-11-14T21:50:05,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741828_1004 (size=1189) 2024-11-14T21:50:05,122 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store 2024-11-14T21:50:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741829_1005 (size=34) 2024-11-14T21:50:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741829_1005 (size=34) 2024-11-14T21:50:05,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741829_1005 (size=34) 2024-11-14T21:50:05,157 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T21:50:05,160 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:05,162 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T21:50:05,162 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:05,162 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:05,164 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T21:50:05,164 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:05,164 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:05,166 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731621005162Disabling compacts and flushes for region at 1731621005162Disabling writes for close at 1731621005164 (+2 ms)Writing region close event to WAL at 1731621005164Closed at 1731621005164 2024-11-14T21:50:05,168 WARN [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/.initializing 2024-11-14T21:50:05,168 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/WALs/ef788a4cade8,37243,1731621003759 2024-11-14T21:50:05,178 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:05,196 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C37243%2C1731621003759, suffix=, logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/WALs/ef788a4cade8,37243,1731621003759, archiveDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/oldWALs, maxLogs=10 2024-11-14T21:50:05,231 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/WALs/ef788a4cade8,37243,1731621003759/ef788a4cade8%2C37243%2C1731621003759.1731621005203, exclude list is [], retry=0 2024-11-14T21:50:05,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:05,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:05,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:05,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:05,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T21:50:05,321 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/WALs/ef788a4cade8,37243,1731621003759/ef788a4cade8%2C37243%2C1731621003759.1731621005203 2024-11-14T21:50:05,322 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:05,322 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:05,323 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:05,326 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,327 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T21:50:05,409 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:05,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,416 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T21:50:05,417 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:05,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T21:50:05,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:05,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T21:50:05,425 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:05,426 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,430 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,431 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,436 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,437 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,440 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T21:50:05,444 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T21:50:05,448 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:05,450 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70201351, jitterRate=0.04608164727687836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T21:50:05,456 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731621005341Initializing all the Stores at 1731621005344 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621005345 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621005346 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621005347 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621005347Cleaning up temporary data from old regions at 1731621005437 (+90 ms)Region opened successfully at 1731621005456 (+19 ms) 2024-11-14T21:50:05,457 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T21:50:05,492 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1771a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef788a4cade8/172.17.0.2:0 2024-11-14T21:50:05,524 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T21:50:05,535 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T21:50:05,535 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T21:50:05,538 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T21:50:05,539 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T21:50:05,544 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T21:50:05,544 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T21:50:05,571 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T21:50:05,579 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T21:50:05,583 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T21:50:05,586 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T21:50:05,587 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T21:50:05,589 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T21:50:05,591 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T21:50:05,594 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T21:50:05,596 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T21:50:05,597 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T21:50:05,599 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T21:50:05,616 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T21:50:05,619 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,628 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef788a4cade8,37243,1731621003759, sessionid=0x100ac6b2eed0000, setting cluster-up flag (Was=false) 2024-11-14T21:50:05,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,648 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T21:50:05,650 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef788a4cade8,37243,1731621003759 2024-11-14T21:50:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:05,664 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T21:50:05,665 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef788a4cade8,37243,1731621003759 2024-11-14T21:50:05,671 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T21:50:05,709 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(746): ClusterId : 5db27ac6-dee0-4c18-b507-d6be882807e2 2024-11-14T21:50:05,709 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(746): ClusterId : 5db27ac6-dee0-4c18-b507-d6be882807e2 2024-11-14T21:50:05,710 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(746): ClusterId : 5db27ac6-dee0-4c18-b507-d6be882807e2 2024-11-14T21:50:05,712 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T21:50:05,712 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T21:50:05,712 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T21:50:05,719 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T21:50:05,719 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T21:50:05,719 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T21:50:05,719 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T21:50:05,720 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T21:50:05,720 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T21:50:05,723 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T21:50:05,723 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T21:50:05,723 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T21:50:05,724 DEBUG [RS:0;ef788a4cade8:39395 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16dcc132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef788a4cade8/172.17.0.2:0 2024-11-14T21:50:05,724 DEBUG [RS:1;ef788a4cade8:36817 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30993faf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef788a4cade8/172.17.0.2:0 2024-11-14T21:50:05,724 DEBUG [RS:2;ef788a4cade8:36167 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c76ae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef788a4cade8/172.17.0.2:0 2024-11-14T21:50:05,745 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ef788a4cade8:36817 2024-11-14T21:50:05,745 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef788a4cade8:39395 2024-11-14T21:50:05,746 DEBUG [RS:2;ef788a4cade8:36167 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ef788a4cade8:36167 2024-11-14T21:50:05,749 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T21:50:05,749 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T21:50:05,749 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T21:50:05,749 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T21:50:05,749 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T21:50:05,749 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T21:50:05,749 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T21:50:05,749 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T21:50:05,749 DEBUG [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T21:50:05,752 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef788a4cade8,37243,1731621003759 with port=39395, startcode=1731621004622 2024-11-14T21:50:05,752 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef788a4cade8,37243,1731621003759 with port=36817, startcode=1731621004730 2024-11-14T21:50:05,752 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef788a4cade8,37243,1731621003759 with port=36167, startcode=1731621004776 2024-11-14T21:50:05,753 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T21:50:05,765 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T21:50:05,768 DEBUG [RS:1;ef788a4cade8:36817 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T21:50:05,768 DEBUG [RS:2;ef788a4cade8:36167 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T21:50:05,768 DEBUG [RS:0;ef788a4cade8:39395 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T21:50:05,776 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T21:50:05,783 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef788a4cade8,37243,1731621003759 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T21:50:05,791 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:50:05,792 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:50:05,792 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:50:05,792 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef788a4cade8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T21:50:05,792 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef788a4cade8:0, corePoolSize=10, maxPoolSize=10 2024-11-14T21:50:05,793 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,793 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef788a4cade8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T21:50:05,793 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,801 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731621035800 2024-11-14T21:50:05,803 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T21:50:05,805 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T21:50:05,810 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T21:50:05,811 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T21:50:05,811 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T21:50:05,811 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T21:50:05,811 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T21:50:05,811 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T21:50:05,817 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59495, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T21:50:05,817 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36033, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T21:50:05,817 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55119, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T21:50:05,814 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,821 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,821 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T21:50:05,822 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T21:50:05,824 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T21:50:05,824 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T21:50:05,828 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T21:50:05,828 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T21:50:05,828 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T21:50:05,831 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.large.0-1731621005830,5,FailOnTimeoutGroup] 2024-11-14T21:50:05,831 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.small.0-1731621005831,5,FailOnTimeoutGroup] 2024-11-14T21:50:05,832 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,832 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T21:50:05,833 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,834 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,836 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef788a4cade8,36817,1731621004730 2024-11-14T21:50:05,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] master.ServerManager(517): Registering regionserver=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:05,850 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6 2024-11-14T21:50:05,850 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45311 2024-11-14T21:50:05,850 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T21:50:05,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:50:05,856 DEBUG [RS:1;ef788a4cade8:36817 {}] zookeeper.ZKUtil(111): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef788a4cade8,36817,1731621004730 2024-11-14T21:50:05,857 WARN [RS:1;ef788a4cade8:36817 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T21:50:05,857 INFO [RS:1;ef788a4cade8:36817 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:05,857 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730 2024-11-14T21:50:05,861 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef788a4cade8,36817,1731621004730] 2024-11-14T21:50:05,866 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef788a4cade8,36167,1731621004776 2024-11-14T21:50:05,867 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] master.ServerManager(517): Registering regionserver=ef788a4cade8,36167,1731621004776 2024-11-14T21:50:05,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741831_1007 (size=1321) 2024-11-14T21:50:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741831_1007 (size=1321) 2024-11-14T21:50:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741831_1007 (size=1321) 2024-11-14T21:50:05,872 DEBUG [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6 2024-11-14T21:50:05,873 DEBUG [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45311 2024-11-14T21:50:05,873 DEBUG [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T21:50:05,875 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T21:50:05,875 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6 2024-11-14T21:50:05,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:50:05,884 DEBUG [RS:2;ef788a4cade8:36167 {}] zookeeper.ZKUtil(111): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef788a4cade8,36167,1731621004776 2024-11-14T21:50:05,884 WARN [RS:2;ef788a4cade8:36167 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T21:50:05,884 INFO [RS:2;ef788a4cade8:36167 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:05,885 DEBUG [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36167,1731621004776 2024-11-14T21:50:05,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef788a4cade8,36167,1731621004776] 2024-11-14T21:50:05,885 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T21:50:05,886 WARN [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T21:50:05,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741832_1008 (size=32) 2024-11-14T21:50:05,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741832_1008 (size=32) 2024-11-14T21:50:05,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741832_1008 (size=32) 2024-11-14T21:50:05,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:05,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T21:50:05,909 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T21:50:05,909 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T21:50:05,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T21:50:05,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:05,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T21:50:05,915 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T21:50:05,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:05,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T21:50:05,920 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T21:50:05,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:05,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T21:50:05,924 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T21:50:05,924 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T21:50:05,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T21:50:05,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:05,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:05,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T21:50:05,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740 2024-11-14T21:50:05,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740 2024-11-14T21:50:05,931 INFO [RS:1;ef788a4cade8:36817 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T21:50:05,931 INFO [RS:2;ef788a4cade8:36167 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T21:50:05,931 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,931 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,932 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T21:50:05,932 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T21:50:05,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T21:50:05,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T21:50:05,935 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T21:50:05,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T21:50:05,940 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T21:50:05,940 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T21:50:05,942 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,942 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,943 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,943 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:05,943 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:50:05,944 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,944 DEBUG [RS:1;ef788a4cade8:36817 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:50:05,945 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,945 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:05,945 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:50:05,945 DEBUG [RS:2;ef788a4cade8:36167 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:50:05,945 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73215795, jitterRate=0.09100036323070526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T21:50:05,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731621005904Initializing all the Stores at 1731621005906 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621005906Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621005907 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621005907Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621005907Cleaning up temporary data from old regions at 1731621005933 (+26 ms)Region opened successfully at 1731621005946 (+13 ms) 2024-11-14T21:50:05,946 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,949 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T21:50:05,949 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,949 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T21:50:05,950 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T21:50:05,950 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T21:50:05,950 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T21:50:05,950 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,950 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,36817,1731621004730-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T21:50:05,950 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,36167,1731621004776-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T21:50:05,954 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T21:50:05,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731621005949Disabling compacts and flushes for region at 1731621005949Disabling writes for close at 1731621005950 (+1 ms)Writing region close event to WAL at 1731621005953 (+3 ms)Closed at 1731621005954 (+1 ms) 2024-11-14T21:50:05,959 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T21:50:05,960 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T21:50:05,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T21:50:05,974 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T21:50:05,974 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T21:50:05,977 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T21:50:05,978 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,36167,1731621004776-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,978 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,978 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,36817,1731621004730-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,978 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.Replication(171): ef788a4cade8,36167,1731621004776 started 2024-11-14T21:50:05,979 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:05,979 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.Replication(171): ef788a4cade8,36817,1731621004730 started 2024-11-14T21:50:05,981 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T21:50:05,987 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef788a4cade8,37243,1731621003759 with port=39395, startcode=1731621004622 2024-11-14T21:50:05,988 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef788a4cade8,39395,1731621004622 2024-11-14T21:50:05,988 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] master.ServerManager(517): Registering regionserver=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:05,992 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6 2024-11-14T21:50:05,992 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45311 2024-11-14T21:50:05,992 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T21:50:05,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:50:05,995 DEBUG [RS:0;ef788a4cade8:39395 {}] zookeeper.ZKUtil(111): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef788a4cade8,39395,1731621004622 2024-11-14T21:50:05,995 WARN [RS:0;ef788a4cade8:39395 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T21:50:05,996 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef788a4cade8,39395,1731621004622] 2024-11-14T21:50:05,996 INFO [RS:0;ef788a4cade8:39395 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:05,996 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622 2024-11-14T21:50:06,003 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T21:50:06,006 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,006 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,006 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1482): Serving as ef788a4cade8,36817,1731621004730, RpcServer on ef788a4cade8/172.17.0.2:36817, sessionid=0x100ac6b2eed0002 2024-11-14T21:50:06,006 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(1482): Serving as ef788a4cade8,36167,1731621004776, RpcServer on ef788a4cade8/172.17.0.2:36167, sessionid=0x100ac6b2eed0003 2024-11-14T21:50:06,007 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T21:50:06,007 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T21:50:06,008 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T21:50:06,008 DEBUG [RS:2;ef788a4cade8:36167 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef788a4cade8,36167,1731621004776 2024-11-14T21:50:06,008 DEBUG [RS:1;ef788a4cade8:36817 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef788a4cade8,36817,1731621004730 2024-11-14T21:50:06,008 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,36817,1731621004730' 2024-11-14T21:50:06,008 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,36167,1731621004776' 2024-11-14T21:50:06,008 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T21:50:06,008 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T21:50:06,008 INFO [RS:0;ef788a4cade8:39395 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T21:50:06,008 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,009 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T21:50:06,009 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T21:50:06,009 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T21:50:06,010 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T21:50:06,011 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T21:50:06,011 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T21:50:06,011 DEBUG [RS:1;ef788a4cade8:36817 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef788a4cade8,36817,1731621004730 2024-11-14T21:50:06,011 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T21:50:06,011 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T21:50:06,011 DEBUG [RS:2;ef788a4cade8:36167 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef788a4cade8,36167,1731621004776 2024-11-14T21:50:06,011 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,36817,1731621004730' 2024-11-14T21:50:06,011 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,36167,1731621004776' 2024-11-14T21:50:06,011 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T21:50:06,011 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T21:50:06,011 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,011 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,011 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,012 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,012 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T21:50:06,012 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,012 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T21:50:06,012 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,012 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T21:50:06,012 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,013 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,013 DEBUG [RS:2;ef788a4cade8:36167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T21:50:06,013 DEBUG [RS:1;ef788a4cade8:36817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T21:50:06,013 INFO [RS:2;ef788a4cade8:36167 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T21:50:06,013 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,013 INFO [RS:1;ef788a4cade8:36817 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T21:50:06,013 INFO [RS:2;ef788a4cade8:36167 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T21:50:06,013 INFO [RS:1;ef788a4cade8:36817 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T21:50:06,013 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,013 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,013 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef788a4cade8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T21:50:06,013 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:50:06,014 DEBUG [RS:0;ef788a4cade8:39395 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef788a4cade8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T21:50:06,022 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,023 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,023 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,023 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,023 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,023 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,39395,1731621004622-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T21:50:06,042 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T21:50:06,042 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,39395,1731621004622-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,042 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,042 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.Replication(171): ef788a4cade8,39395,1731621004622 started 2024-11-14T21:50:06,059 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:06,059 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1482): Serving as ef788a4cade8,39395,1731621004622, RpcServer on ef788a4cade8/172.17.0.2:39395, sessionid=0x100ac6b2eed0001 2024-11-14T21:50:06,059 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T21:50:06,060 DEBUG [RS:0;ef788a4cade8:39395 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef788a4cade8,39395,1731621004622 2024-11-14T21:50:06,060 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,39395,1731621004622' 2024-11-14T21:50:06,060 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T21:50:06,061 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T21:50:06,061 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T21:50:06,061 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T21:50:06,061 DEBUG [RS:0;ef788a4cade8:39395 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef788a4cade8,39395,1731621004622 2024-11-14T21:50:06,061 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef788a4cade8,39395,1731621004622' 2024-11-14T21:50:06,061 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T21:50:06,062 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T21:50:06,063 DEBUG [RS:0;ef788a4cade8:39395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T21:50:06,063 INFO [RS:0;ef788a4cade8:39395 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T21:50:06,063 INFO [RS:0;ef788a4cade8:39395 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T21:50:06,118 INFO [RS:1;ef788a4cade8:36817 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:06,118 INFO [RS:2;ef788a4cade8:36167 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:06,122 INFO [RS:1;ef788a4cade8:36817 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C36817%2C1731621004730, suffix=, logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730, archiveDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs, maxLogs=32 2024-11-14T21:50:06,122 INFO [RS:2;ef788a4cade8:36167 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C36167%2C1731621004776, suffix=, logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36167,1731621004776, archiveDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs, maxLogs=32 2024-11-14T21:50:06,132 WARN [ef788a4cade8:37243 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T21:50:06,140 DEBUG [RS:1;ef788a4cade8:36817 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730/ef788a4cade8%2C36817%2C1731621004730.1731621006126, exclude list is [], retry=0 2024-11-14T21:50:06,140 DEBUG [RS:2;ef788a4cade8:36167 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36167,1731621004776/ef788a4cade8%2C36167%2C1731621004776.1731621006126, exclude list is [], retry=0 2024-11-14T21:50:06,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:06,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:06,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:06,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:06,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:06,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:06,155 INFO [RS:2;ef788a4cade8:36167 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36167,1731621004776/ef788a4cade8%2C36167%2C1731621004776.1731621006126 2024-11-14T21:50:06,155 INFO [RS:1;ef788a4cade8:36817 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730/ef788a4cade8%2C36817%2C1731621004730.1731621006126 2024-11-14T21:50:06,157 DEBUG [RS:2;ef788a4cade8:36167 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:06,158 DEBUG [RS:1;ef788a4cade8:36817 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:06,164 INFO [RS:0;ef788a4cade8:39395 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:06,167 INFO [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C39395%2C1731621004622, suffix=, logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622, archiveDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs, maxLogs=32 2024-11-14T21:50:06,187 DEBUG [RS:0;ef788a4cade8:39395 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622/ef788a4cade8%2C39395%2C1731621004622.1731621006168, exclude list is [], retry=0 2024-11-14T21:50:06,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:06,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:06,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:06,198 INFO [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622/ef788a4cade8%2C39395%2C1731621004622.1731621006168 2024-11-14T21:50:06,199 DEBUG [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:06,385 DEBUG [ef788a4cade8:37243 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T21:50:06,396 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(204): Hosts are {ef788a4cade8=0} racks are {/default-rack=0} 2024-11-14T21:50:06,404 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T21:50:06,405 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T21:50:06,405 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T21:50:06,405 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T21:50:06,405 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T21:50:06,405 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T21:50:06,405 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T21:50:06,405 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T21:50:06,405 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T21:50:06,405 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T21:50:06,415 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:06,531 INFO [AsyncFSWAL-0-hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData-prefix:ef788a4cade8,37243,1731621003759 {}] wal.AbstractFSWAL(1368): Slow sync cost: 113 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK], DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK], DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK]] 2024-11-14T21:50:06,535 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef788a4cade8,36817,1731621004730, state=OPENING 2024-11-14T21:50:06,566 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T21:50:06,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:06,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:06,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:06,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:06,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,592 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T21:50:06,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef788a4cade8,36817,1731621004730}] 2024-11-14T21:50:06,793 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T21:50:06,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35721, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T21:50:06,807 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T21:50:06,807 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:06,808 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T21:50:06,811 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C36817%2C1731621004730.meta, suffix=.meta, logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730, archiveDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs, maxLogs=32 2024-11-14T21:50:06,827 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, exclude list is [], retry=0 2024-11-14T21:50:06,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:06,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:06,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:06,835 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta 2024-11-14T21:50:06,836 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:06,836 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:06,837 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T21:50:06,840 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T21:50:06,846 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T21:50:06,850 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T21:50:06,851 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:06,852 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T21:50:06,852 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T21:50:06,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T21:50:06,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T21:50:06,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:06,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:06,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T21:50:06,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T21:50:06,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:06,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:06,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T21:50:06,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T21:50:06,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:06,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:06,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T21:50:06,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T21:50:06,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:06,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:06,867 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T21:50:06,868 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740 2024-11-14T21:50:06,871 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740 2024-11-14T21:50:06,873 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T21:50:06,873 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T21:50:06,874 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T21:50:06,877 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T21:50:06,879 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75478602, jitterRate=0.1247188150882721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T21:50:06,879 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T21:50:06,880 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731621006852Writing region info on filesystem at 1731621006852Initializing all the Stores at 1731621006854 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621006854Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621006855 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621006855Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621006855Cleaning up temporary data from old regions at 1731621006873 (+18 ms)Running coprocessor post-open hooks at 1731621006879 (+6 ms)Region opened successfully at 1731621006880 (+1 ms) 2024-11-14T21:50:06,888 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731621006785 2024-11-14T21:50:06,899 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T21:50:06,900 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T21:50:06,901 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:06,904 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef788a4cade8,36817,1731621004730, state=OPEN 2024-11-14T21:50:06,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:06,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:06,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:06,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:06,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:06,907 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:06,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T21:50:06,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef788a4cade8,36817,1731621004730 in 313 msec 2024-11-14T21:50:06,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T21:50:06,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 948 msec 2024-11-14T21:50:06,921 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T21:50:06,921 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T21:50:06,941 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:06,942 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1] 2024-11-14T21:50:06,963 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:50:06,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49495, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:50:06,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2860 sec 2024-11-14T21:50:06,986 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731621006986, completionTime=-1 2024-11-14T21:50:06,988 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T21:50:06,989 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T21:50:07,018 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-14T21:50:07,019 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731621067019 2024-11-14T21:50:07,019 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731621127019 2024-11-14T21:50:07,019 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-14T21:50:07,021 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T21:50:07,028 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37243,1731621003759-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,029 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37243,1731621003759-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,029 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37243,1731621003759-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,031 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef788a4cade8:37243, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,031 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,032 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,039 DEBUG [master/ef788a4cade8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T21:50:07,067 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.205sec 2024-11-14T21:50:07,069 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T21:50:07,071 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T21:50:07,072 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T21:50:07,073 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T21:50:07,073 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T21:50:07,074 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37243,1731621003759-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T21:50:07,075 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37243,1731621003759-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T21:50:07,080 DEBUG [master/ef788a4cade8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T21:50:07,081 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T21:50:07,082 INFO [master/ef788a4cade8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef788a4cade8,37243,1731621003759-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:07,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8ccb42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:50:07,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef788a4cade8,37243,-1 for getting cluster id 2024-11-14T21:50:07,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T21:50:07,138 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5db27ac6-dee0-4c18-b507-d6be882807e2' 2024-11-14T21:50:07,141 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T21:50:07,142 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5db27ac6-dee0-4c18-b507-d6be882807e2" 2024-11-14T21:50:07,145 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a56a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:50:07,145 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef788a4cade8,37243,-1] 2024-11-14T21:50:07,148 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T21:50:07,151 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:07,152 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T21:50:07,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ce80ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T21:50:07,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:07,166 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1] 2024-11-14T21:50:07,167 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:50:07,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:50:07,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef788a4cade8,37243,1731621003759 2024-11-14T21:50:07,199 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:45311/hbase 2024-11-14T21:50:07,217 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=557, ProcessCount=11, AvailableMemoryMB=5315 2024-11-14T21:50:07,238 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:07,242 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:07,244 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:07,250 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99102774, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-99102774, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:07,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99102774/hregion-99102774.1731621007252, exclude list is [], retry=0 2024-11-14T21:50:07,279 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:07,279 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:07,280 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:07,289 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99102774/hregion-99102774.1731621007252 2024-11-14T21:50:07,290 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:07,290 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b752fcdfb3770edf028d908faa3a8e27, NAME => 'testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:07,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741838_1014 (size=64) 2024-11-14T21:50:07,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741838_1014 (size=64) 2024-11-14T21:50:07,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741838_1014 (size=64) 2024-11-14T21:50:07,313 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:07,323 INFO [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,327 INFO [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b752fcdfb3770edf028d908faa3a8e27 columnFamilyName a 2024-11-14T21:50:07,327 DEBUG [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:07,328 INFO [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] regionserver.HStore(327): Store=b752fcdfb3770edf028d908faa3a8e27/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:07,328 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,332 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,332 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,337 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:07,343 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:07,346 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b752fcdfb3770edf028d908faa3a8e27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65627115, jitterRate=-0.02207978069782257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:07,347 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b752fcdfb3770edf028d908faa3a8e27: Writing region info on filesystem at 1731621007314Initializing all the Stores at 1731621007317 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621007317Cleaning up temporary data from old regions at 1731621007332 (+15 ms)Region opened successfully at 1731621007347 (+15 ms) 2024-11-14T21:50:07,347 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b752fcdfb3770edf028d908faa3a8e27, disabling compactions & flushes 2024-11-14T21:50:07,347 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27. 2024-11-14T21:50:07,347 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27. 2024-11-14T21:50:07,347 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27. after waiting 0 ms 2024-11-14T21:50:07,347 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27. 2024-11-14T21:50:07,351 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27. 2024-11-14T21:50:07,351 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b752fcdfb3770edf028d908faa3a8e27: Waiting for close lock at 1731621007347Disabling compacts and flushes for region at 1731621007347Disabling writes for close at 1731621007347Writing region close event to WAL at 1731621007350 (+3 ms)Closed at 1731621007351 (+1 ms) 2024-11-14T21:50:07,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741837_1013 (size=93) 2024-11-14T21:50:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741837_1013 (size=93) 2024-11-14T21:50:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741837_1013 (size=93) 2024-11-14T21:50:07,364 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:07,364 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99102774:(num 1731621007252) 2024-11-14T21:50:07,366 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T21:50:07,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741839_1015 (size=369) 2024-11-14T21:50:07,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741839_1015 (size=369) 2024-11-14T21:50:07,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741839_1015 (size=369) 2024-11-14T21:50:07,400 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T21:50:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741840_1016 (size=231) 2024-11-14T21:50:07,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741840_1016 (size=231) 2024-11-14T21:50:07,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741840_1016 (size=231) 2024-11-14T21:50:07,448 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1, size=369 (369bytes) 2024-11-14T21:50:07,450 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T21:50:07,450 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T21:50:07,450 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1 2024-11-14T21:50:07,458 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1 after 6ms 2024-11-14T21:50:07,467 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1 took 21ms 2024-11-14T21:50:07,473 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1 so closing down 2024-11-14T21:50:07,477 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T21:50:07,479 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:07,479 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T21:50:07,480 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:07,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741841_1017 (size=369) 2024-11-14T21:50:07,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741841_1017 (size=369) 2024-11-14T21:50:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741841_1017 (size=369) 2024-11-14T21:50:07,498 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:07,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002 2024-11-14T21:50:07,508 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-14T21:50:07,508 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1, journal: Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1, size=369 (369bytes) at 1731621007448Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1 so closing down at 1731621007473 (+25 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000001-wal-1.temp at 1731621007479 (+6 ms)3 split writer threads finished at 1731621007480 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731621007498 (+18 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002 at 1731621007502 (+4 ms)Processed 2 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731621007508 (+6 ms) 2024-11-14T21:50:07,535 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2, size=231 (231bytes) 2024-11-14T21:50:07,535 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2 2024-11-14T21:50:07,536 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2 after 1ms 2024-11-14T21:50:07,543 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2 took 9ms 2024-11-14T21:50:07,547 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2 so closing down 2024-11-14T21:50:07,548 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:07,555 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T21:50:07,559 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T21:50:07,559 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:07,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741842_1018 (size=231) 2024-11-14T21:50:07,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741842_1018 (size=231) 2024-11-14T21:50:07,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741842_1018 (size=231) 2024-11-14T21:50:07,984 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:07,992 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-14T21:50:07,994 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 450 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-14T21:50:07,994 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2, journal: Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2, size=231 (231bytes) at 1731621007535Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2 so closing down at 1731621007547 (+12 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002-wal-2.temp at 1731621007559 (+12 ms)3 split writer threads finished at 1731621007559Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731621007984 (+425 ms)Processed 1 edits across 1 Regions in 450 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731621007994 (+10 ms) 2024-11-14T21:50:07,995 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:07,998 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:08,014 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal.1731621007999, exclude list is [], retry=0 2024-11-14T21:50:08,018 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:08,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:08,020 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:08,030 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal.1731621007999 2024-11-14T21:50:08,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:08,031 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b752fcdfb3770edf028d908faa3a8e27, NAME => 'testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:08,031 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:08,031 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,031 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,035 INFO [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,037 INFO [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b752fcdfb3770edf028d908faa3a8e27 columnFamilyName a 2024-11-14T21:50:08,037 DEBUG [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:08,038 INFO [StoreOpener-b752fcdfb3770edf028d908faa3a8e27-1 {}] regionserver.HStore(327): Store=b752fcdfb3770edf028d908faa3a8e27/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:08,038 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,043 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,045 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002 2024-11-14T21:50:08,056 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002 2024-11-14T21:50:08,060 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b752fcdfb3770edf028d908faa3a8e27 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T21:50:08,129 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/.tmp/a/86e8b9067df2463c8942cfb0a557fb6e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731621007364/Put/seqid=0 2024-11-14T21:50:08,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741844_1020 (size=5170) 2024-11-14T21:50:08,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741844_1020 (size=5170) 2024-11-14T21:50:08,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741844_1020 (size=5170) 2024-11-14T21:50:08,151 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/.tmp/a/86e8b9067df2463c8942cfb0a557fb6e 2024-11-14T21:50:08,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/.tmp/a/86e8b9067df2463c8942cfb0a557fb6e as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/a/86e8b9067df2463c8942cfb0a557fb6e 2024-11-14T21:50:08,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/a/86e8b9067df2463c8942cfb0a557fb6e, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T21:50:08,285 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b752fcdfb3770edf028d908faa3a8e27 in 222ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T21:50:08,287 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/0000000000000000002 2024-11-14T21:50:08,291 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,291 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,296 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b752fcdfb3770edf028d908faa3a8e27 2024-11-14T21:50:08,302 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b752fcdfb3770edf028d908faa3a8e27/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T21:50:08,308 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b752fcdfb3770edf028d908faa3a8e27; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59405173, jitterRate=-0.11479394137859344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:08,310 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b752fcdfb3770edf028d908faa3a8e27: Writing region info on filesystem at 1731621008031Initializing all the Stores at 1731621008033 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621008033Obtaining lock to block concurrent updates at 1731621008061 (+28 ms)Preparing flush snapshotting stores in b752fcdfb3770edf028d908faa3a8e27 at 1731621008061Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731621008066 (+5 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731621007239.b752fcdfb3770edf028d908faa3a8e27. at 1731621008066Flushing b752fcdfb3770edf028d908faa3a8e27/a: creating writer at 1731621008068 (+2 ms)Flushing b752fcdfb3770edf028d908faa3a8e27/a: appending metadata at 1731621008119 (+51 ms)Flushing b752fcdfb3770edf028d908faa3a8e27/a: closing flushed file at 1731621008122 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ed7bf9c: reopening flushed file at 1731621008246 (+124 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b752fcdfb3770edf028d908faa3a8e27 in 222ms, sequenceid=2, compaction requested=false; wal=null at 1731621008285 (+39 ms)Cleaning up temporary data from old regions at 1731621008291 (+6 ms)Region opened successfully at 1731621008309 (+18 ms) 2024-11-14T21:50:08,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=379 (was 368) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:33480 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:33324 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:53700 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:45311/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:53856 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:43904 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=699 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=557 (was 557), ProcessCount=11 (was 11), AvailableMemoryMB=5101 (was 5315) 2024-11-14T21:50:08,358 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=379, OpenFileDescriptor=699, MaxFileDescriptor=1048576, SystemLoadAverage=557, ProcessCount=11, AvailableMemoryMB=5101 2024-11-14T21:50:08,383 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:08,387 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:08,388 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:08,393 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49809792, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-49809792, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:08,415 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49809792/hregion-49809792.1731621008394, exclude list is [], retry=0 2024-11-14T21:50:08,422 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:08,423 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:08,425 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:08,435 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49809792/hregion-49809792.1731621008394 2024-11-14T21:50:08,436 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:08,436 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 687f2e45f692f824ef4a3ee1dbbb648c, NAME => 'testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:08,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741846_1022 (size=64) 2024-11-14T21:50:08,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741846_1022 (size=64) 2024-11-14T21:50:08,484 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:08,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741846_1022 (size=64) 2024-11-14T21:50:08,488 INFO [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,494 INFO [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 687f2e45f692f824ef4a3ee1dbbb648c columnFamilyName a 2024-11-14T21:50:08,494 DEBUG [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:08,495 INFO [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] regionserver.HStore(327): Store=687f2e45f692f824ef4a3ee1dbbb648c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:08,495 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,497 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,498 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,499 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,499 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,502 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,507 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:08,507 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 687f2e45f692f824ef4a3ee1dbbb648c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68776678, jitterRate=0.024852365255355835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:08,508 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 687f2e45f692f824ef4a3ee1dbbb648c: Writing region info on filesystem at 1731621008484Initializing all the Stores at 1731621008486 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621008486Cleaning up temporary data from old regions at 1731621008499 (+13 ms)Region opened successfully at 1731621008508 (+9 ms) 2024-11-14T21:50:08,508 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 687f2e45f692f824ef4a3ee1dbbb648c, disabling compactions & flushes 2024-11-14T21:50:08,508 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c. 2024-11-14T21:50:08,508 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c. 2024-11-14T21:50:08,508 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c. after waiting 0 ms 2024-11-14T21:50:08,508 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c. 2024-11-14T21:50:08,511 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c. 2024-11-14T21:50:08,511 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 687f2e45f692f824ef4a3ee1dbbb648c: Waiting for close lock at 1731621008508Disabling compacts and flushes for region at 1731621008508Disabling writes for close at 1731621008508Writing region close event to WAL at 1731621008511 (+3 ms)Closed at 1731621008511 2024-11-14T21:50:08,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741845_1021 (size=93) 2024-11-14T21:50:08,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741845_1021 (size=93) 2024-11-14T21:50:08,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741845_1021 (size=93) 2024-11-14T21:50:08,545 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:08,545 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49809792:(num 1731621008394) 2024-11-14T21:50:08,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T21:50:08,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741847_1023 (size=369) 2024-11-14T21:50:08,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741847_1023 (size=369) 2024-11-14T21:50:08,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741847_1023 (size=369) 2024-11-14T21:50:08,582 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T21:50:08,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741848_1024 (size=231) 2024-11-14T21:50:08,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741848_1024 (size=231) 2024-11-14T21:50:08,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741848_1024 (size=231) 2024-11-14T21:50:08,622 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2, size=231 (231bytes) 2024-11-14T21:50:08,622 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2 2024-11-14T21:50:08,623 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2 after 1ms 2024-11-14T21:50:08,628 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2 took 6ms 2024-11-14T21:50:08,632 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2 so closing down 2024-11-14T21:50:08,632 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:08,635 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T21:50:08,637 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T21:50:08,638 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:08,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741849_1025 (size=231) 2024-11-14T21:50:08,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741849_1025 (size=231) 2024-11-14T21:50:08,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741849_1025 (size=231) 2024-11-14T21:50:08,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:08,659 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 2024-11-14T21:50:08,659 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-14T21:50:08,659 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2, journal: Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2, size=231 (231bytes) at 1731621008622Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2 so closing down at 1731621008632 (+10 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002-wal-2.temp at 1731621008637 (+5 ms)3 split writer threads finished at 1731621008638 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731621008655 (+17 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 at 1731621008659 (+4 ms)Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731621008659 2024-11-14T21:50:08,684 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1, size=369 (369bytes) 2024-11-14T21:50:08,684 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1 2024-11-14T21:50:08,685 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1 after 1ms 2024-11-14T21:50:08,690 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1 took 6ms 2024-11-14T21:50:08,693 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1 so closing down 2024-11-14T21:50:08,693 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:08,699 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T21:50:08,702 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T21:50:08,702 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:08,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741850_1026 (size=369) 2024-11-14T21:50:08,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741850_1026 (size=369) 2024-11-14T21:50:08,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741850_1026 (size=369) 2024-11-14T21:50:08,726 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:08,735 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002, length=231 2024-11-14T21:50:08,742 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 2024-11-14T21:50:08,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-14T21:50:08,743 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1, journal: Splitting hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1, size=369 (369bytes) at 1731621008684Finishing writing output for hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1 so closing down at 1731621008693 (+9 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000001-wal-1.temp at 1731621008702 (+9 ms)3 split writer threads finished at 1731621008702Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731621008726 (+24 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 at 1731621008742 (+16 ms)Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731621008743 (+1 ms) 2024-11-14T21:50:08,743 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:08,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:08,770 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal.1731621008749, exclude list is [], retry=0 2024-11-14T21:50:08,777 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:08,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:08,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:08,787 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal.1731621008749 2024-11-14T21:50:08,788 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:08,788 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 687f2e45f692f824ef4a3ee1dbbb648c, NAME => 'testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:08,789 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:08,789 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,789 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,792 INFO [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,794 INFO [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 687f2e45f692f824ef4a3ee1dbbb648c columnFamilyName a 2024-11-14T21:50:08,794 DEBUG [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:08,799 INFO [StoreOpener-687f2e45f692f824ef4a3ee1dbbb648c-1 {}] regionserver.HStore(327): Store=687f2e45f692f824ef4a3ee1dbbb648c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:08,800 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,801 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,805 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:08,806 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 2024-11-14T21:50:08,811 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 2024-11-14T21:50:08,811 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 687f2e45f692f824ef4a3ee1dbbb648c 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T21:50:08,830 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/.tmp/a/22297e36d3a44cb2af654bb499872132 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731621008545/Put/seqid=0 2024-11-14T21:50:08,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741852_1028 (size=5170) 2024-11-14T21:50:08,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741852_1028 (size=5170) 2024-11-14T21:50:08,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741852_1028 (size=5170) 2024-11-14T21:50:09,247 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/.tmp/a/22297e36d3a44cb2af654bb499872132 2024-11-14T21:50:09,257 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/.tmp/a/22297e36d3a44cb2af654bb499872132 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/a/22297e36d3a44cb2af654bb499872132 2024-11-14T21:50:09,268 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/a/22297e36d3a44cb2af654bb499872132, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T21:50:09,269 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 687f2e45f692f824ef4a3ee1dbbb648c in 458ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T21:50:09,270 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/0000000000000000002 2024-11-14T21:50:09,270 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:09,271 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:09,274 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 687f2e45f692f824ef4a3ee1dbbb648c 2024-11-14T21:50:09,277 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/687f2e45f692f824ef4a3ee1dbbb648c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T21:50:09,278 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 687f2e45f692f824ef4a3ee1dbbb648c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59743812, jitterRate=-0.10974782705307007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:09,278 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 687f2e45f692f824ef4a3ee1dbbb648c: Writing region info on filesystem at 1731621008789Initializing all the Stores at 1731621008791 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621008791Obtaining lock to block concurrent updates at 1731621008811 (+20 ms)Preparing flush snapshotting stores in 687f2e45f692f824ef4a3ee1dbbb648c at 1731621008811Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731621008811Flushing stores of testReplayEditsWrittenIntoWAL,,1731621008384.687f2e45f692f824ef4a3ee1dbbb648c. at 1731621008811Flushing 687f2e45f692f824ef4a3ee1dbbb648c/a: creating writer at 1731621008811Flushing 687f2e45f692f824ef4a3ee1dbbb648c/a: appending metadata at 1731621008829 (+18 ms)Flushing 687f2e45f692f824ef4a3ee1dbbb648c/a: closing flushed file at 1731621008829Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0303aa: reopening flushed file at 1731621009255 (+426 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 687f2e45f692f824ef4a3ee1dbbb648c in 458ms, sequenceid=2, compaction requested=false; wal=null at 1731621009269 (+14 ms)Cleaning up temporary data from old regions at 1731621009271 (+2 ms)Region opened successfully at 1731621009278 (+7 ms) 2024-11-14T21:50:09,309 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 379) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:55364 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:53700 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:33324 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:47142 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:51818 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:55340 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 699) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=557 (was 557), ProcessCount=11 (was 11), AvailableMemoryMB=5029 (was 5101) 2024-11-14T21:50:09,324 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=557, ProcessCount=11, AvailableMemoryMB=5017 2024-11-14T21:50:09,349 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:09,352 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:09,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:09,357 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-90171609, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-90171609, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:09,374 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-90171609/hregion-90171609.1731621009358, exclude list is [], retry=0 2024-11-14T21:50:09,379 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:09,379 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:09,379 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:09,382 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-90171609/hregion-90171609.1731621009358 2024-11-14T21:50:09,383 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:09,384 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b158fa1af224247a4ef679bef2f8dbe3, NAME => 'testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:09,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741854_1030 (size=64) 2024-11-14T21:50:09,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741854_1030 (size=64) 2024-11-14T21:50:09,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741854_1030 (size=64) 2024-11-14T21:50:09,398 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:09,400 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,402 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b158fa1af224247a4ef679bef2f8dbe3 columnFamilyName a 2024-11-14T21:50:09,403 DEBUG [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:09,403 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(327): Store=b158fa1af224247a4ef679bef2f8dbe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:09,403 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,405 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b158fa1af224247a4ef679bef2f8dbe3 columnFamilyName b 2024-11-14T21:50:09,405 DEBUG [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:09,406 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(327): Store=b158fa1af224247a4ef679bef2f8dbe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:09,406 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,409 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b158fa1af224247a4ef679bef2f8dbe3 columnFamilyName c 2024-11-14T21:50:09,409 DEBUG [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:09,409 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(327): Store=b158fa1af224247a4ef679bef2f8dbe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:09,410 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,413 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,413 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,414 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:09,416 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:09,420 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:09,420 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b158fa1af224247a4ef679bef2f8dbe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63152316, jitterRate=-0.05895715951919556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:09,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b158fa1af224247a4ef679bef2f8dbe3: Writing region info on filesystem at 1731621009398Initializing all the Stores at 1731621009400 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621009400Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621009400Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621009400Cleaning up temporary data from old regions at 1731621009413 (+13 ms)Region opened successfully at 1731621009420 (+7 ms) 2024-11-14T21:50:09,421 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b158fa1af224247a4ef679bef2f8dbe3, disabling compactions & flushes 2024-11-14T21:50:09,421 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:09,421 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:09,421 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. after waiting 0 ms 2024-11-14T21:50:09,421 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:09,422 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:09,422 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b158fa1af224247a4ef679bef2f8dbe3: Waiting for close lock at 1731621009421Disabling compacts and flushes for region at 1731621009421Disabling writes for close at 1731621009421Writing region close event to WAL at 1731621009421Closed at 1731621009421 2024-11-14T21:50:09,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741853_1029 (size=93) 2024-11-14T21:50:09,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741853_1029 (size=93) 2024-11-14T21:50:09,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741853_1029 (size=93) 2024-11-14T21:50:09,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:09,431 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-90171609:(num 1731621009358) 2024-11-14T21:50:09,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:09,434 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:09,454 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435, exclude list is [], retry=0 2024-11-14T21:50:09,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:09,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:09,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:09,462 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 2024-11-14T21:50:09,462 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:09,684 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435, size=0 (0bytes) 2024-11-14T21:50:09,684 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 might be still open, length is 0 2024-11-14T21:50:09,684 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 2024-11-14T21:50:09,685 WARN [IPC Server handler 2 on default port 45311 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-14T21:50:09,686 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 after 2ms 2024-11-14T21:50:11,686 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:51842 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:32913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51842 dst: /127.0.0.1:32913 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:32913 remote=/127.0.0.1:51842]. Total timeout mills is 60000, 57961 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:11,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:55396 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39681:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55396 dst: /127.0.0.1:39681 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:11,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:47178 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44645:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47178 dst: /127.0.0.1:44645 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:11,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741855_1032 (size=470675) 2024-11-14T21:50:11,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741855_1032 (size=470675) 2024-11-14T21:50:11,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741855_1032 (size=470675) 2024-11-14T21:50:12,361 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:50:12,425 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T21:50:13,687 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 after 4003ms 2024-11-14T21:50:13,694 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 took 4010ms 2024-11-14T21:50:13,704 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731621009435.temp 2024-11-14T21:50:13,722 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000000001-wal.1731621009435.temp 2024-11-14T21:50:13,817 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435; continuing. 2024-11-14T21:50:13,817 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 so closing down 2024-11-14T21:50:13,817 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:13,819 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741856_1033 (size=470683) 2024-11-14T21:50:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741856_1033 (size=470683) 2024-11-14T21:50:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741856_1033 (size=470683) 2024-11-14T21:50:13,831 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000000001-wal.1731621009435.temp (wrote 3002 edits, skipped 0 edits in 33 ms) 2024-11-14T21:50:13,838 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000000001-wal.1731621009435.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000003002 2024-11-14T21:50:13,839 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 144 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T21:50:13,840 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435, journal: Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435, size=0 (0bytes) at 1731621009684Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000000001-wal.1731621009435.temp at 1731621013722 (+4038 ms)Split 1024 edits, skipped 0 edits. at 1731621013784 (+62 ms)Split 2048 edits, skipped 0 edits. at 1731621013805 (+21 ms)Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 so closing down at 1731621013817 (+12 ms)3 split writer threads finished at 1731621013819 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000000001-wal.1731621009435.temp (wrote 3002 edits, skipped 0 edits in 33 ms) at 1731621013831 (+12 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000000001-wal.1731621009435.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000003002 at 1731621013839 (+8 ms)Processed 3002 edits across 1 Regions in 144 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435, size=0, length=0, corrupted=false, cancelled=false at 1731621013839 2024-11-14T21:50:13,856 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621009435 2024-11-14T21:50:13,861 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000003002 2024-11-14T21:50:13,861 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:13,869 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:13,894 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621013869, exclude list is [], retry=0 2024-11-14T21:50:13,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:13,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:13,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:13,916 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621013869 2024-11-14T21:50:13,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:13,917 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:13,921 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:13,923 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b158fa1af224247a4ef679bef2f8dbe3 columnFamilyName a 2024-11-14T21:50:13,923 DEBUG [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:13,924 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(327): Store=b158fa1af224247a4ef679bef2f8dbe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:13,924 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:13,926 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b158fa1af224247a4ef679bef2f8dbe3 columnFamilyName b 2024-11-14T21:50:13,926 DEBUG [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:13,927 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(327): Store=b158fa1af224247a4ef679bef2f8dbe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:13,927 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:13,934 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b158fa1af224247a4ef679bef2f8dbe3 columnFamilyName c 2024-11-14T21:50:13,934 DEBUG [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:13,938 INFO [StoreOpener-b158fa1af224247a4ef679bef2f8dbe3-1 {}] regionserver.HStore(327): Store=b158fa1af224247a4ef679bef2f8dbe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:13,940 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:13,944 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:13,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:13,950 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000003002 2024-11-14T21:50:14,014 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T21:50:14,265 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T21:50:14,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:14,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T21:50:14,267 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T21:50:14,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T21:50:14,268 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:14,269 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-14T21:50:14,269 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:14,634 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b158fa1af224247a4ef679bef2f8dbe3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T21:50:14,751 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/a/7082258af0664dc09a3f41c29bf6e7e8 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731621009468/Put/seqid=0 2024-11-14T21:50:14,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741858_1035 (size=50463) 2024-11-14T21:50:14,821 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/a/7082258af0664dc09a3f41c29bf6e7e8 2024-11-14T21:50:14,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741858_1035 (size=50463) 2024-11-14T21:50:14,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741858_1035 (size=50463) 2024-11-14T21:50:14,876 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/a/7082258af0664dc09a3f41c29bf6e7e8 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/a/7082258af0664dc09a3f41c29bf6e7e8 2024-11-14T21:50:14,907 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/a/7082258af0664dc09a3f41c29bf6e7e8, entries=754, sequenceid=754, filesize=49.3 K 2024-11-14T21:50:14,907 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for b158fa1af224247a4ef679bef2f8dbe3 in 274ms, sequenceid=754, compaction requested=false; wal=null 2024-11-14T21:50:14,932 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T21:50:14,932 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b158fa1af224247a4ef679bef2f8dbe3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T21:50:14,971 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/a/491db17a7cce453ca03d72ee5d0dfa1d is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731621009520/Put/seqid=0 2024-11-14T21:50:15,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741859_1036 (size=20072) 2024-11-14T21:50:15,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741859_1036 (size=20072) 2024-11-14T21:50:15,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741859_1036 (size=20072) 2024-11-14T21:50:15,028 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/a/491db17a7cce453ca03d72ee5d0dfa1d 2024-11-14T21:50:15,077 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/b/874fee5a63ef42a2bd7b1f2c45e1254f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731621009545/Put/seqid=0 2024-11-14T21:50:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741860_1037 (size=35835) 2024-11-14T21:50:15,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741860_1037 (size=35835) 2024-11-14T21:50:15,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741860_1037 (size=35835) 2024-11-14T21:50:15,105 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/b/874fee5a63ef42a2bd7b1f2c45e1254f 2024-11-14T21:50:15,118 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/a/491db17a7cce453ca03d72ee5d0dfa1d as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/a/491db17a7cce453ca03d72ee5d0dfa1d 2024-11-14T21:50:15,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/a/491db17a7cce453ca03d72ee5d0dfa1d, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-14T21:50:15,143 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/b/874fee5a63ef42a2bd7b1f2c45e1254f as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/b/874fee5a63ef42a2bd7b1f2c45e1254f 2024-11-14T21:50:15,154 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/b/874fee5a63ef42a2bd7b1f2c45e1254f, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-14T21:50:15,155 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for b158fa1af224247a4ef679bef2f8dbe3 in 223ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-14T21:50:15,163 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T21:50:15,163 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b158fa1af224247a4ef679bef2f8dbe3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T21:50:15,176 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/b/2ecb58a6188c440095fe76753a3eee16 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731621009567/Put/seqid=0 2024-11-14T21:50:15,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741861_1038 (size=35082) 2024-11-14T21:50:15,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741861_1038 (size=35082) 2024-11-14T21:50:15,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741861_1038 (size=35082) 2024-11-14T21:50:15,251 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/b/2ecb58a6188c440095fe76753a3eee16 2024-11-14T21:50:15,317 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/c/fd0781f8775d4d7db0866fda02b675a4 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731621009594/Put/seqid=0 2024-11-14T21:50:15,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741862_1039 (size=20825) 2024-11-14T21:50:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741862_1039 (size=20825) 2024-11-14T21:50:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741862_1039 (size=20825) 2024-11-14T21:50:15,367 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/c/fd0781f8775d4d7db0866fda02b675a4 2024-11-14T21:50:15,381 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/b/2ecb58a6188c440095fe76753a3eee16 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/b/2ecb58a6188c440095fe76753a3eee16 2024-11-14T21:50:15,398 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/b/2ecb58a6188c440095fe76753a3eee16, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-14T21:50:15,406 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/c/fd0781f8775d4d7db0866fda02b675a4 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/c/fd0781f8775d4d7db0866fda02b675a4 2024-11-14T21:50:15,427 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/c/fd0781f8775d4d7db0866fda02b675a4, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-14T21:50:15,428 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for b158fa1af224247a4ef679bef2f8dbe3 in 265ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-14T21:50:15,437 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731621009645/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:15,441 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000003002 2024-11-14T21:50:15,442 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T21:50:15,442 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b158fa1af224247a4ef679bef2f8dbe3 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-14T21:50:15,455 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/c/2003782eba184360bc960f97dfa052e2 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731621009602/Put/seqid=0 2024-11-14T21:50:15,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741863_1040 (size=50301) 2024-11-14T21:50:15,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741863_1040 (size=50301) 2024-11-14T21:50:15,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741863_1040 (size=50301) 2024-11-14T21:50:15,513 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/c/2003782eba184360bc960f97dfa052e2 2024-11-14T21:50:15,526 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2003782eba184360bc960f97dfa052e2 2024-11-14T21:50:15,535 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/.tmp/c/2003782eba184360bc960f97dfa052e2 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/c/2003782eba184360bc960f97dfa052e2 2024-11-14T21:50:15,562 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2003782eba184360bc960f97dfa052e2 2024-11-14T21:50:15,562 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/c/2003782eba184360bc960f97dfa052e2, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-14T21:50:15,562 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for b158fa1af224247a4ef679bef2f8dbe3 in 120ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-14T21:50:15,566 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/0000000000000003002 2024-11-14T21:50:15,574 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:15,575 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:15,580 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T21:50:15,583 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b158fa1af224247a4ef679bef2f8dbe3 2024-11-14T21:50:15,587 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenIntoWAL/b158fa1af224247a4ef679bef2f8dbe3/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-14T21:50:15,588 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b158fa1af224247a4ef679bef2f8dbe3; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70381851, jitterRate=0.04877130687236786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T21:50:15,589 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b158fa1af224247a4ef679bef2f8dbe3: Writing region info on filesystem at 1731621013917Initializing all the Stores at 1731621013919 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621013919Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621013921 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621013921Cleaning up temporary data from old regions at 1731621015575 (+1654 ms)Region opened successfully at 1731621015589 (+14 ms) 2024-11-14T21:50:15,711 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b158fa1af224247a4ef679bef2f8dbe3, disabling compactions & flushes 2024-11-14T21:50:15,711 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:15,711 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:15,711 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. after waiting 0 ms 2024-11-14T21:50:15,711 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:15,754 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731621009350.b158fa1af224247a4ef679bef2f8dbe3. 2024-11-14T21:50:15,754 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b158fa1af224247a4ef679bef2f8dbe3: Waiting for close lock at 1731621015711Disabling compacts and flushes for region at 1731621015711Disabling writes for close at 1731621015711Writing region close event to WAL at 1731621015754 (+43 ms)Closed at 1731621015754 2024-11-14T21:50:15,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741857_1034 (size=93) 2024-11-14T21:50:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741857_1034 (size=93) 2024-11-14T21:50:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741857_1034 (size=93) 2024-11-14T21:50:15,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:15,781 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731621013869) 2024-11-14T21:50:15,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=406 (was 390) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:45311 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_519753378_22 at /127.0.0.1:55482 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:44607 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33265 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33265 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45311 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_519753378_22 at /127.0.0.1:55450 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_519753378_22 at /127.0.0.1:47226 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44607 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_519753378_22 at /127.0.0.1:51864 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=591 (was 557) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3778 (was 5017) 2024-11-14T21:50:15,829 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=406, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=591, ProcessCount=11, AvailableMemoryMB=3763 2024-11-14T21:50:15,856 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:15,861 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:15,863 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:15,874 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-04322210, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-04322210, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:15,896 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-04322210/hregion-04322210.1731621015875, exclude list is [], retry=0 2024-11-14T21:50:15,901 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:15,904 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:15,910 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:15,928 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-04322210/hregion-04322210.1731621015875 2024-11-14T21:50:15,929 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:15,929 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5b00a6e50008b6e3ee481734f95e93ae, NAME => 'test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741865_1042 (size=43) 2024-11-14T21:50:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741865_1042 (size=43) 2024-11-14T21:50:15,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741865_1042 (size=43) 2024-11-14T21:50:16,376 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:16,378 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,380 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b00a6e50008b6e3ee481734f95e93ae columnFamilyName a 2024-11-14T21:50:16,381 DEBUG [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:16,381 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(327): Store=5b00a6e50008b6e3ee481734f95e93ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:16,382 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,384 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b00a6e50008b6e3ee481734f95e93ae columnFamilyName b 2024-11-14T21:50:16,384 DEBUG [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:16,385 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(327): Store=5b00a6e50008b6e3ee481734f95e93ae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:16,385 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,387 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b00a6e50008b6e3ee481734f95e93ae columnFamilyName c 2024-11-14T21:50:16,387 DEBUG [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:16,388 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(327): Store=5b00a6e50008b6e3ee481734f95e93ae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:16,388 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,389 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,392 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,392 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,394 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:16,396 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:16,399 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:16,399 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b00a6e50008b6e3ee481734f95e93ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73623426, jitterRate=0.09707453846931458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:16,400 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b00a6e50008b6e3ee481734f95e93ae: Writing region info on filesystem at 1731621016376Initializing all the Stores at 1731621016378 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621016378Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621016378Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621016378Cleaning up temporary data from old regions at 1731621016392 (+14 ms)Region opened successfully at 1731621016400 (+8 ms) 2024-11-14T21:50:16,401 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b00a6e50008b6e3ee481734f95e93ae, disabling compactions & flushes 2024-11-14T21:50:16,401 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:16,401 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:16,401 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. after waiting 0 ms 2024-11-14T21:50:16,401 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:16,403 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:16,403 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b00a6e50008b6e3ee481734f95e93ae: Waiting for close lock at 1731621016401Disabling compacts and flushes for region at 1731621016401Disabling writes for close at 1731621016401Writing region close event to WAL at 1731621016403 (+2 ms)Closed at 1731621016403 2024-11-14T21:50:16,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741864_1041 (size=93) 2024-11-14T21:50:16,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741864_1041 (size=93) 2024-11-14T21:50:16,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741864_1041 (size=93) 2024-11-14T21:50:16,412 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:16,412 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-04322210:(num 1731621015875) 2024-11-14T21:50:16,413 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:16,417 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:16,436 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417, exclude list is [], retry=0 2024-11-14T21:50:16,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:16,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:16,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:16,515 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 2024-11-14T21:50:16,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:16,726 INFO [AsyncFSWAL-0-hdfs://localhost:45311/hbase-prefix:default {}] wal.AbstractFSWAL(1368): Slow sync cost: 102 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK], DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK], DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK]] 2024-11-14T21:50:16,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741866_1043 (size=344306) 2024-11-14T21:50:16,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741866_1043 (size=344306) 2024-11-14T21:50:16,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741866_1043 (size=344306) 2024-11-14T21:50:16,863 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417, size=336.2 K (344306bytes) 2024-11-14T21:50:16,863 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 2024-11-14T21:50:16,864 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 after 1ms 2024-11-14T21:50:16,868 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 took 5ms 2024-11-14T21:50:16,872 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731621016417.temp 2024-11-14T21:50:16,875 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000000001-wal.1731621016417.temp 2024-11-14T21:50:16,897 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 so closing down 2024-11-14T21:50:16,898 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:16,898 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:16,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741867_1044 (size=344306) 2024-11-14T21:50:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741867_1044 (size=344306) 2024-11-14T21:50:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741867_1044 (size=344306) 2024-11-14T21:50:16,941 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000000001-wal.1731621016417.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-11-14T21:50:16,946 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000000001-wal.1731621016417.temp to hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003000 2024-11-14T21:50:16,947 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-14T21:50:16,947 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417, journal: Splitting hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417, size=336.2 K (344306bytes) at 1731621016863Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000000001-wal.1731621016417.temp at 1731621016875 (+12 ms)Split 1024 edits, skipped 0 edits. at 1731621016879 (+4 ms)Split 2048 edits, skipped 0 edits. at 1731621016887 (+8 ms)Finishing writing output for hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 so closing down at 1731621016898 (+11 ms)3 split writer threads finished at 1731621016898Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000000001-wal.1731621016417.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1731621016941 (+43 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000000001-wal.1731621016417.temp to hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003000 at 1731621016946 (+5 ms)Processed 3000 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731621016947 (+1 ms) 2024-11-14T21:50:16,949 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016417 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621016417 2024-11-14T21:50:16,951 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003000 2024-11-14T21:50:16,951 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:16,955 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:16,974 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956, exclude list is [], retry=0 2024-11-14T21:50:16,979 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:16,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:16,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:16,991 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 2024-11-14T21:50:16,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:17,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741868_1045 (size=344433) 2024-11-14T21:50:17,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741868_1045 (size=344433) 2024-11-14T21:50:17,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741868_1045 (size=344433) 2024-11-14T21:50:17,283 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956, size=336.4 K (344433bytes) 2024-11-14T21:50:17,284 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 2024-11-14T21:50:17,285 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 after 0ms 2024-11-14T21:50:17,292 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 took 8ms 2024-11-14T21:50:17,309 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731621016956.temp 2024-11-14T21:50:17,316 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003001-wal.1731621016956.temp 2024-11-14T21:50:17,344 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 so closing down 2024-11-14T21:50:17,344 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:17,344 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:17,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741869_1046 (size=344433) 2024-11-14T21:50:17,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741869_1046 (size=344433) 2024-11-14T21:50:17,427 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003001-wal.1731621016956.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-11-14T21:50:17,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741869_1046 (size=344433) 2024-11-14T21:50:17,430 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003001-wal.1731621016956.temp to hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000006000 2024-11-14T21:50:17,431 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 136 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-14T21:50:17,431 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956, journal: Splitting hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956, size=336.4 K (344433bytes) at 1731621017283Split 1024 edits, skipped 0 edits. at 1731621017310 (+27 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003001-wal.1731621016956.temp at 1731621017316 (+6 ms)Split 2048 edits, skipped 0 edits. at 1731621017328 (+12 ms)Finishing writing output for hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 so closing down at 1731621017344 (+16 ms)3 split writer threads finished at 1731621017344Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003001-wal.1731621016956.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1731621017427 (+83 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003001-wal.1731621016956.temp to hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000006000 at 1731621017431 (+4 ms)Processed 3000 edits across 1 Regions in 136 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731621017431 2024-11-14T21:50:17,436 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621016956 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621016956 2024-11-14T21:50:17,439 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000006000 2024-11-14T21:50:17,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:17,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/test2727-manual,16010,1731621015855, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:17,462 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621017442, exclude list is [], retry=0 2024-11-14T21:50:17,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:17,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:17,468 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:17,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621017442 2024-11-14T21:50:17,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:17,480 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b00a6e50008b6e3ee481734f95e93ae, NAME => 'test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:17,481 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:17,481 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,481 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,484 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,486 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b00a6e50008b6e3ee481734f95e93ae columnFamilyName a 2024-11-14T21:50:17,486 DEBUG [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:17,487 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(327): Store=5b00a6e50008b6e3ee481734f95e93ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:17,488 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,492 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b00a6e50008b6e3ee481734f95e93ae columnFamilyName b 2024-11-14T21:50:17,492 DEBUG [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:17,495 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(327): Store=5b00a6e50008b6e3ee481734f95e93ae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:17,496 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,497 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b00a6e50008b6e3ee481734f95e93ae columnFamilyName c 2024-11-14T21:50:17,497 DEBUG [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:17,500 INFO [StoreOpener-5b00a6e50008b6e3ee481734f95e93ae-1 {}] regionserver.HStore(327): Store=5b00a6e50008b6e3ee481734f95e93ae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:17,500 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:17,506 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003000 2024-11-14T21:50:17,579 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003000 2024-11-14T21:50:17,591 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000006000 2024-11-14T21:50:17,643 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000006000 2024-11-14T21:50:17,643 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b00a6e50008b6e3ee481734f95e93ae 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-14T21:50:17,686 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/a/32b45dbcc1e44c698dbf6b2d6466977e is 41, key is test2727/a:100/1731621017005/Put/seqid=0 2024-11-14T21:50:17,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741871_1048 (size=84227) 2024-11-14T21:50:17,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741871_1048 (size=84227) 2024-11-14T21:50:17,728 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/a/32b45dbcc1e44c698dbf6b2d6466977e 2024-11-14T21:50:17,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741871_1048 (size=84227) 2024-11-14T21:50:17,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/b/49aad6650db44899b9ecf32fd0da4868 is 41, key is test2727/b:100/1731621017057/Put/seqid=0 2024-11-14T21:50:17,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741872_1049 (size=84609) 2024-11-14T21:50:17,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741872_1049 (size=84609) 2024-11-14T21:50:17,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741872_1049 (size=84609) 2024-11-14T21:50:17,830 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/b/49aad6650db44899b9ecf32fd0da4868 2024-11-14T21:50:17,870 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/c/409df64b868444939d31d55080474951 is 41, key is test2727/c:100/1731621017176/Put/seqid=0 2024-11-14T21:50:17,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741873_1050 (size=84609) 2024-11-14T21:50:17,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741873_1050 (size=84609) 2024-11-14T21:50:17,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741873_1050 (size=84609) 2024-11-14T21:50:17,920 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/c/409df64b868444939d31d55080474951 2024-11-14T21:50:17,932 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/a/32b45dbcc1e44c698dbf6b2d6466977e as hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/a/32b45dbcc1e44c698dbf6b2d6466977e 2024-11-14T21:50:17,957 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/a/32b45dbcc1e44c698dbf6b2d6466977e, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-14T21:50:17,959 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/b/49aad6650db44899b9ecf32fd0da4868 as hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/b/49aad6650db44899b9ecf32fd0da4868 2024-11-14T21:50:17,982 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/b/49aad6650db44899b9ecf32fd0da4868, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T21:50:17,984 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/.tmp/c/409df64b868444939d31d55080474951 as hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/c/409df64b868444939d31d55080474951 2024-11-14T21:50:18,002 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/c/409df64b868444939d31d55080474951, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T21:50:18,002 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 5b00a6e50008b6e3ee481734f95e93ae in 359ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-14T21:50:18,004 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000003000 2024-11-14T21:50:18,004 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/0000000000000006000 2024-11-14T21:50:18,007 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:18,007 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:18,008 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:18,011 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b00a6e50008b6e3ee481734f95e93ae 2024-11-14T21:50:18,020 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/test2727/5b00a6e50008b6e3ee481734f95e93ae/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-14T21:50:18,025 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b00a6e50008b6e3ee481734f95e93ae; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64447925, jitterRate=-0.03965108096599579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:18,027 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b00a6e50008b6e3ee481734f95e93ae: Writing region info on filesystem at 1731621017481Initializing all the Stores at 1731621017483 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621017483Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621017484 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621017484Obtaining lock to block concurrent updates at 1731621017643 (+159 ms)Preparing flush snapshotting stores in 5b00a6e50008b6e3ee481734f95e93ae at 1731621017643Finished memstore snapshotting test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731621017643Flushing stores of test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. at 1731621017643Flushing 5b00a6e50008b6e3ee481734f95e93ae/a: creating writer at 1731621017643Flushing 5b00a6e50008b6e3ee481734f95e93ae/a: appending metadata at 1731621017685 (+42 ms)Flushing 5b00a6e50008b6e3ee481734f95e93ae/a: closing flushed file at 1731621017685Flushing 5b00a6e50008b6e3ee481734f95e93ae/b: creating writer at 1731621017741 (+56 ms)Flushing 5b00a6e50008b6e3ee481734f95e93ae/b: appending metadata at 1731621017802 (+61 ms)Flushing 5b00a6e50008b6e3ee481734f95e93ae/b: closing flushed file at 1731621017802Flushing 5b00a6e50008b6e3ee481734f95e93ae/c: creating writer at 1731621017839 (+37 ms)Flushing 5b00a6e50008b6e3ee481734f95e93ae/c: appending metadata at 1731621017869 (+30 ms)Flushing 5b00a6e50008b6e3ee481734f95e93ae/c: closing flushed file at 1731621017869Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f230d45: reopening flushed file at 1731621017929 (+60 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20325925: reopening flushed file at 1731621017958 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c801069: reopening flushed file at 1731621017982 (+24 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 5b00a6e50008b6e3ee481734f95e93ae in 359ms, sequenceid=6000, compaction requested=false; wal=null at 1731621018002 (+20 ms)Cleaning up temporary data from old regions at 1731621018007 (+5 ms)Region opened successfully at 1731621018026 (+19 ms) 2024-11-14T21:50:18,028 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-14T21:50:18,028 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b00a6e50008b6e3ee481734f95e93ae, disabling compactions & flushes 2024-11-14T21:50:18,028 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:18,028 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:18,028 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. after waiting 0 ms 2024-11-14T21:50:18,029 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:18,051 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731621015858.5b00a6e50008b6e3ee481734f95e93ae. 2024-11-14T21:50:18,051 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b00a6e50008b6e3ee481734f95e93ae: Waiting for close lock at 1731621018028Disabling compacts and flushes for region at 1731621018028Disabling writes for close at 1731621018028Writing region close event to WAL at 1731621018050 (+22 ms)Closed at 1731621018050 2024-11-14T21:50:18,059 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731621015855/wal.1731621017442 not finished, retry = 0 2024-11-14T21:50:18,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741870_1047 (size=93) 2024-11-14T21:50:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741870_1047 (size=93) 2024-11-14T21:50:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741870_1047 (size=93) 2024-11-14T21:50:18,164 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:18,164 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731621017442) 2024-11-14T21:50:18,181 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 406), OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=591 (was 591), ProcessCount=11 (was 11), AvailableMemoryMB=2365 (was 3763) 2024-11-14T21:50:18,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=591, ProcessCount=11, AvailableMemoryMB=2357 2024-11-14T21:50:18,214 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:18,221 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:18,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731621018222 2024-11-14T21:50:18,230 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222 2024-11-14T21:50:18,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:18,233 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a8b82e2050c022a8dc538271f1b69334, NAME => 'testSequentialEditLogSeqNum,,1731621018215.a8b82e2050c022a8dc538271f1b69334.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:18,234 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731621018215.a8b82e2050c022a8dc538271f1b69334.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:18,234 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,234 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,235 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334 doesn't exist for region: a8b82e2050c022a8dc538271f1b69334 on table testSequentialEditLogSeqNum 2024-11-14T21:50:18,235 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: a8b82e2050c022a8dc538271f1b69334 on table testSequentialEditLogSeqNum 2024-11-14T21:50:18,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741875_1052 (size=62) 2024-11-14T21:50:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741875_1052 (size=62) 2024-11-14T21:50:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741875_1052 (size=62) 2024-11-14T21:50:18,247 INFO [StoreOpener-a8b82e2050c022a8dc538271f1b69334-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,248 INFO [StoreOpener-a8b82e2050c022a8dc538271f1b69334-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8b82e2050c022a8dc538271f1b69334 columnFamilyName a 2024-11-14T21:50:18,248 DEBUG [StoreOpener-a8b82e2050c022a8dc538271f1b69334-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,249 INFO [StoreOpener-a8b82e2050c022a8dc538271f1b69334-1 {}] regionserver.HStore(327): Store=a8b82e2050c022a8dc538271f1b69334/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,249 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,250 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,250 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,251 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,251 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,253 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a8b82e2050c022a8dc538271f1b69334 2024-11-14T21:50:18,256 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:18,256 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a8b82e2050c022a8dc538271f1b69334; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61193647, jitterRate=-0.08814360201358795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:18,258 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a8b82e2050c022a8dc538271f1b69334: Writing region info on filesystem at 1731621018234Initializing all the Stores at 1731621018246 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018246Cleaning up temporary data from old regions at 1731621018251 (+5 ms)Region opened successfully at 1731621018258 (+7 ms) 2024-11-14T21:50:18,270 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a8b82e2050c022a8dc538271f1b69334 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-14T21:50:18,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/.tmp/a/807c07f6f9c94804a3c264e70ef4ebad is 81, key is testSequentialEditLogSeqNum/a:x0/1731621018258/Put/seqid=0 2024-11-14T21:50:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741876_1053 (size=5833) 2024-11-14T21:50:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741876_1053 (size=5833) 2024-11-14T21:50:18,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741876_1053 (size=5833) 2024-11-14T21:50:18,361 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/.tmp/a/807c07f6f9c94804a3c264e70ef4ebad 2024-11-14T21:50:18,368 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/.tmp/a/807c07f6f9c94804a3c264e70ef4ebad as hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/a/807c07f6f9c94804a3c264e70ef4ebad 2024-11-14T21:50:18,375 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/a/807c07f6f9c94804a3c264e70ef4ebad, entries=10, sequenceid=13, filesize=5.7 K 2024-11-14T21:50:18,376 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a8b82e2050c022a8dc538271f1b69334 in 106ms, sequenceid=13, compaction requested=false 2024-11-14T21:50:18,376 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a8b82e2050c022a8dc538271f1b69334: 2024-11-14T21:50:18,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T21:50:18,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T21:50:18,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T21:50:18,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T21:50:18,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T21:50:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741874_1051 (size=3017) 2024-11-14T21:50:18,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741874_1051 (size=3017) 2024-11-14T21:50:18,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741874_1051 (size=3017) 2024-11-14T21:50:18,410 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222, size=2.9 K (3017bytes) 2024-11-14T21:50:18,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222 2024-11-14T21:50:18,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222 after 0ms 2024-11-14T21:50:18,414 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222 took 4ms 2024-11-14T21:50:18,417 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222 so closing down 2024-11-14T21:50:18,417 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:18,419 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731621018222.temp 2024-11-14T21:50:18,473 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000003-wal.1731621018222.temp 2024-11-14T21:50:18,473 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:18,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741877_1054 (size=2433) 2024-11-14T21:50:18,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741877_1054 (size=2433) 2024-11-14T21:50:18,504 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000003-wal.1731621018222.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:18,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741877_1054 (size=2433) 2024-11-14T21:50:18,506 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000003-wal.1731621018222.temp to hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000020 2024-11-14T21:50:18,507 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 92 ms; skipped=2; WAL=hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-14T21:50:18,507 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222, journal: Splitting hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222, size=2.9 K (3017bytes) at 1731621018411Finishing writing output for hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222 so closing down at 1731621018417 (+6 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000003-wal.1731621018222.temp at 1731621018473 (+56 ms)3 split writer threads finished at 1731621018473Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000003-wal.1731621018222.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731621018504 (+31 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000003-wal.1731621018222.temp to hdfs://localhost:45311/hbase/data/default/testSequentialEditLogSeqNum/a8b82e2050c022a8dc538271f1b69334/recovered.edits/0000000000000000020 at 1731621018506 (+2 ms)Processed 17 edits across 1 Regions in 92 ms; skipped=2; WAL=hdfs://localhost:45311/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731621018213/wal.1731621018222, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731621018507 (+1 ms) 2024-11-14T21:50:18,532 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:47280 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:55450 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=591 (was 591), ProcessCount=11 (was 11), AvailableMemoryMB=2243 (was 2357) 2024-11-14T21:50:18,546 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=591, ProcessCount=11, AvailableMemoryMB=2238 2024-11-14T21:50:18,564 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:18,566 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:18,591 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:18,594 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85424554, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-85424554, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:18,611 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85424554/hregion-85424554.1731621018595, exclude list is [], retry=0 2024-11-14T21:50:18,615 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:18,615 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:18,616 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:18,666 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85424554/hregion-85424554.1731621018595 2024-11-14T21:50:18,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:18,667 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3eddfeaeeb483caa5c479e1e02a6db75, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:18,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741879_1056 (size=70) 2024-11-14T21:50:18,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741879_1056 (size=70) 2024-11-14T21:50:18,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741879_1056 (size=70) 2024-11-14T21:50:18,683 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:18,685 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,686 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName a 2024-11-14T21:50:18,686 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,687 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,687 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,689 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName b 2024-11-14T21:50:18,689 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,689 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,689 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,691 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName c 2024-11-14T21:50:18,691 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,691 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,691 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,692 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,694 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,694 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,695 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:18,698 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,701 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:18,701 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3eddfeaeeb483caa5c479e1e02a6db75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63252525, jitterRate=-0.05746392905712128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3eddfeaeeb483caa5c479e1e02a6db75: Writing region info on filesystem at 1731621018683Initializing all the Stores at 1731621018684 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018684Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018684Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018684Cleaning up temporary data from old regions at 1731621018694 (+10 ms)Region opened successfully at 1731621018702 (+8 ms) 2024-11-14T21:50:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3eddfeaeeb483caa5c479e1e02a6db75, disabling compactions & flushes 2024-11-14T21:50:18,702 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:18,702 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. after waiting 0 ms 2024-11-14T21:50:18,703 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:18,703 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:18,703 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3eddfeaeeb483caa5c479e1e02a6db75: Waiting for close lock at 1731621018702Disabling compacts and flushes for region at 1731621018702Disabling writes for close at 1731621018703 (+1 ms)Writing region close event to WAL at 1731621018703Closed at 1731621018703 2024-11-14T21:50:18,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741878_1055 (size=93) 2024-11-14T21:50:18,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741878_1055 (size=93) 2024-11-14T21:50:18,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741878_1055 (size=93) 2024-11-14T21:50:18,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:18,748 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85424554:(num 1731621018595) 2024-11-14T21:50:18,748 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:18,751 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:18,766 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751, exclude list is [], retry=0 2024-11-14T21:50:18,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:18,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:18,771 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:18,773 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 2024-11-14T21:50:18,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:18,773 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3eddfeaeeb483caa5c479e1e02a6db75, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:18,773 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:18,773 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,773 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,775 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,776 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName a 2024-11-14T21:50:18,776 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,777 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,777 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,778 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName b 2024-11-14T21:50:18,778 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,778 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,778 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,779 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName c 2024-11-14T21:50:18,779 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:18,780 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:18,780 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,781 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,782 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,784 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,784 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,784 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:18,786 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:18,787 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3eddfeaeeb483caa5c479e1e02a6db75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69273386, jitterRate=0.03225389122962952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:18,788 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3eddfeaeeb483caa5c479e1e02a6db75: Writing region info on filesystem at 1731621018774Initializing all the Stores at 1731621018775 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018775Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018775Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621018775Cleaning up temporary data from old regions at 1731621018784 (+9 ms)Region opened successfully at 1731621018788 (+4 ms) 2024-11-14T21:50:18,792 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731621018791/Put/seqid=0 2024-11-14T21:50:18,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741881_1058 (size=4826) 2024-11-14T21:50:18,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741881_1058 (size=4826) 2024-11-14T21:50:18,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741881_1058 (size=4826) 2024-11-14T21:50:18,802 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45311/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 3eddfeaeeb483caa5c479e1e02a6db75/a 2024-11-14T21:50:18,814 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-14T21:50:18,814 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T21:50:18,814 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3eddfeaeeb483caa5c479e1e02a6db75: 2024-11-14T21:50:18,816 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_ 2024-11-14T21:50:18,818 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45311/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 3eddfeaeeb483caa5c479e1e02a6db75/a as hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_ - updating store file list. 2024-11-14T21:50:18,824 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:18,825 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_ into 3eddfeaeeb483caa5c479e1e02a6db75/a 2024-11-14T21:50:18,825 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45311/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 3eddfeaeeb483caa5c479e1e02a6db75/a (new location: hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_) 2024-11-14T21:50:18,878 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751, size=0 (0bytes) 2024-11-14T21:50:18,878 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 might be still open, length is 0 2024-11-14T21:50:18,878 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 2024-11-14T21:50:18,878 WARN [IPC Server handler 1 on default port 45311 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-14T21:50:18,879 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 after 1ms 2024-11-14T21:50:20,671 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:36700 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44645:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36700 dst: /127.0.0.1:44645 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44645 remote=/127.0.0.1:36700]. Total timeout mills is 60000, 58161 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:20,671 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:34836 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:32913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34836 dst: /127.0.0.1:32913 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:20,671 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:54636 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39681:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54636 dst: /127.0.0.1:39681 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:20,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741880_1059 (size=504) 2024-11-14T21:50:20,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741880_1059 (size=504) 2024-11-14T21:50:22,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 after 4002ms 2024-11-14T21:50:22,885 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 took 4007ms 2024-11-14T21:50:22,887 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751; continuing. 2024-11-14T21:50:22,887 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 so closing down 2024-11-14T21:50:22,887 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:22,893 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731621018751.temp 2024-11-14T21:50:22,898 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005-wal.1731621018751.temp 2024-11-14T21:50:22,899 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:23,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741882_1060 (size=238) 2024-11-14T21:50:23,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741882_1060 (size=238) 2024-11-14T21:50:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741882_1060 (size=238) 2024-11-14T21:50:23,111 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005-wal.1731621018751.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:23,114 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005-wal.1731621018751.temp to hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005 2024-11-14T21:50:23,115 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 230 ms; skipped=1; WAL=hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T21:50:23,115 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751, journal: Splitting hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751, size=0 (0bytes) at 1731621018878Finishing writing output for hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 so closing down at 1731621022887 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005-wal.1731621018751.temp at 1731621022898 (+11 ms)3 split writer threads finished at 1731621022899 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005-wal.1731621018751.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731621023111 (+212 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005-wal.1731621018751.temp to hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005 at 1731621023114 (+3 ms)Processed 2 edits across 1 Regions in 230 ms; skipped=1; WAL=hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751, size=0, length=0, corrupted=false, cancelled=false at 1731621023115 (+1 ms) 2024-11-14T21:50:23,118 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621018751 2024-11-14T21:50:23,123 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005 2024-11-14T21:50:23,123 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:23,126 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:23,150 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621023127, exclude list is [], retry=0 2024-11-14T21:50:23,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:23,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:23,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:23,157 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621023127 2024-11-14T21:50:23,158 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:23,158 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3eddfeaeeb483caa5c479e1e02a6db75, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:23,158 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:23,158 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,158 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,162 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,163 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName a 2024-11-14T21:50:23,164 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:23,173 DEBUG [StoreFileOpener-3eddfeaeeb483caa5c479e1e02a6db75-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:23,173 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/bdff8f32bf4e4f0685702825dc28dc66_SeqId_3_ 2024-11-14T21:50:23,174 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:23,174 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,181 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName b 2024-11-14T21:50:23,181 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:23,182 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:23,183 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,184 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddfeaeeb483caa5c479e1e02a6db75 columnFamilyName c 2024-11-14T21:50:23,184 DEBUG [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:23,189 INFO [StoreOpener-3eddfeaeeb483caa5c479e1e02a6db75-1 {}] regionserver.HStore(327): Store=3eddfeaeeb483caa5c479e1e02a6db75/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:23,190 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,198 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,202 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005 2024-11-14T21:50:23,211 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005 2024-11-14T21:50:23,211 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3eddfeaeeb483caa5c479e1e02a6db75 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-14T21:50:23,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/.tmp/a/28266b23244d46819c04e8e202255d18 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731621018830/Put/seqid=0 2024-11-14T21:50:23,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741884_1062 (size=5149) 2024-11-14T21:50:23,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741884_1062 (size=5149) 2024-11-14T21:50:23,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741884_1062 (size=5149) 2024-11-14T21:50:23,282 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/.tmp/a/28266b23244d46819c04e8e202255d18 2024-11-14T21:50:23,316 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/.tmp/a/28266b23244d46819c04e8e202255d18 as hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/28266b23244d46819c04e8e202255d18 2024-11-14T21:50:23,325 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/a/28266b23244d46819c04e8e202255d18, entries=1, sequenceid=5, filesize=5.0 K 2024-11-14T21:50:23,325 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 3eddfeaeeb483caa5c479e1e02a6db75 in 114ms, sequenceid=5, compaction requested=false; wal=null 2024-11-14T21:50:23,326 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/0000000000000000005 2024-11-14T21:50:23,328 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,328 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,329 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:23,332 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3eddfeaeeb483caa5c479e1e02a6db75 2024-11-14T21:50:23,340 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3eddfeaeeb483caa5c479e1e02a6db75/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-14T21:50:23,341 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3eddfeaeeb483caa5c479e1e02a6db75; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60528644, jitterRate=-0.09805291891098022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:23,342 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3eddfeaeeb483caa5c479e1e02a6db75: Writing region info on filesystem at 1731621023158Initializing all the Stores at 1731621023160 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621023160Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621023162 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621023162Obtaining lock to block concurrent updates at 1731621023211 (+49 ms)Preparing flush snapshotting stores in 3eddfeaeeb483caa5c479e1e02a6db75 at 1731621023211Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731621023211Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. at 1731621023211Flushing 3eddfeaeeb483caa5c479e1e02a6db75/a: creating writer at 1731621023211Flushing 3eddfeaeeb483caa5c479e1e02a6db75/a: appending metadata at 1731621023236 (+25 ms)Flushing 3eddfeaeeb483caa5c479e1e02a6db75/a: closing flushed file at 1731621023236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16faf182: reopening flushed file at 1731621023315 (+79 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 3eddfeaeeb483caa5c479e1e02a6db75 in 114ms, sequenceid=5, compaction requested=false; wal=null at 1731621023325 (+10 ms)Cleaning up temporary data from old regions at 1731621023328 (+3 ms)Region opened successfully at 1731621023341 (+13 ms) 2024-11-14T21:50:23,350 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3eddfeaeeb483caa5c479e1e02a6db75, disabling compactions & flushes 2024-11-14T21:50:23,350 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:23,350 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:23,350 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. after waiting 0 ms 2024-11-14T21:50:23,350 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:23,354 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731621018565.3eddfeaeeb483caa5c479e1e02a6db75. 2024-11-14T21:50:23,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3eddfeaeeb483caa5c479e1e02a6db75: Waiting for close lock at 1731621023350Disabling compacts and flushes for region at 1731621023350Disabling writes for close at 1731621023350Writing region close event to WAL at 1731621023354 (+4 ms)Closed at 1731621023354 2024-11-14T21:50:23,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741883_1061 (size=93) 2024-11-14T21:50:23,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741883_1061 (size=93) 2024-11-14T21:50:23,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741883_1061 (size=93) 2024-11-14T21:50:23,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:23,367 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731621023127) 2024-11-14T21:50:23,386 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 409) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925084878_22 at /127.0.0.1:36728 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1925084878_22 at /127.0.0.1:34870 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45311 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:45311 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=632 (was 591) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1279 (was 2238) 2024-11-14T21:50:23,406 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=632, ProcessCount=11, AvailableMemoryMB=1277 2024-11-14T21:50:23,432 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:23,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T21:50:23,445 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef788a4cade8,37243,1731621003759 2024-11-14T21:50:23,449 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d128bd5 2024-11-14T21:50:23,450 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T21:50:23,453 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T21:50:23,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T21:50:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T21:50:23,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T21:50:23,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-14T21:50:23,470 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:50:23,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T21:50:23,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741885_1063 (size=694) 2024-11-14T21:50:23,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741885_1063 (size=694) 2024-11-14T21:50:23,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741885_1063 (size=694) 2024-11-14T21:50:23,493 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6 2024-11-14T21:50:23,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741886_1064 (size=77) 2024-11-14T21:50:23,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741886_1064 (size=77) 2024-11-14T21:50:23,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741886_1064 (size=77) 2024-11-14T21:50:23,507 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:23,508 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:23,508 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,508 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,508 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:23,508 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,508 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,508 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621023508Disabling compacts and flushes for region at 1731621023508Disabling writes for close at 1731621023508Writing region close event to WAL at 1731621023508Closed at 1731621023508 2024-11-14T21:50:23,511 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T21:50:23,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731621023511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731621023511"}]},"ts":"1731621023511"} 2024-11-14T21:50:23,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T21:50:23,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T21:50:23,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731621023527"}]},"ts":"1731621023527"} 2024-11-14T21:50:23,535 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-14T21:50:23,535 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ef788a4cade8=0} racks are {/default-rack=0} 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T21:50:23,537 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T21:50:23,537 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T21:50:23,537 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T21:50:23,537 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T21:50:23,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN}] 2024-11-14T21:50:23,540 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN 2024-11-14T21:50:23,542 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN; state=OFFLINE, location=ef788a4cade8,36817,1731621004730; forceNewPlan=false, retain=false 2024-11-14T21:50:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:50:23,694 INFO [ef788a4cade8:37243 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T21:50:23,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPENING, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:23,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN because future has completed 2024-11-14T21:50:23,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730}] 2024-11-14T21:50:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:50:23,862 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,863 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:23,863 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,863 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:23,863 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,863 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,866 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,872 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf1 2024-11-14T21:50:23,872 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:23,873 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:23,873 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,876 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf2 2024-11-14T21:50:23,876 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:23,876 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:23,876 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,878 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,878 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,879 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,879 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,881 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T21:50:23,882 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,886 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:23,886 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 264804e76ef9209d727464aa496291f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74350024, jitterRate=0.1079016923904419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T21:50:23,886 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:23,926 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 264804e76ef9209d727464aa496291f4: Running coprocessor pre-open hook at 1731621023864Writing region info on filesystem at 1731621023864Initializing all the Stores at 1731621023865 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621023865Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621023866 (+1 ms)Cleaning up temporary data from old regions at 1731621023879 (+13 ms)Running coprocessor post-open hooks at 1731621023887 (+8 ms)Region opened successfully at 1731621023926 (+39 ms) 2024-11-14T21:50:23,939 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., pid=6, masterSystemTime=1731621023855 2024-11-14T21:50:23,942 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,942 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:23,943 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPEN, openSeqNum=2, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:23,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730 because future has completed 2024-11-14T21:50:23,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T21:50:23,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730 in 248 msec 2024-11-14T21:50:23,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T21:50:23,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN in 414 msec 2024-11-14T21:50:23,957 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T21:50:23,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731621023957"}]},"ts":"1731621023957"} 2024-11-14T21:50:23,960 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-14T21:50:23,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T21:50:23,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 502 msec 2024-11-14T21:50:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T21:50:24,103 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-14T21:50:24,103 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-14T21:50:24,104 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T21:50:24,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-14T21:50:24,111 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T21:50:24,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-14T21:50:24,126 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=2] 2024-11-14T21:50:24,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=264804e76ef9209d727464aa496291f4, source=ef788a4cade8,36817,1731621004730, destination=ef788a4cade8,39395,1731621004622, warming up region on ef788a4cade8,39395,1731621004622 2024-11-14T21:50:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T21:50:24,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=264804e76ef9209d727464aa496291f4, source=ef788a4cade8,36817,1731621004730, destination=ef788a4cade8,39395,1731621004622, running balancer 2024-11-14T21:50:24,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44823, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T21:50:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE 2024-11-14T21:50:24,156 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE 2024-11-14T21:50:24,158 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=CLOSING, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:24,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(7855): Warmup {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:24,161 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE because future has completed 2024-11-14T21:50:24,162 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf1 2024-11-14T21:50:24,162 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:24,163 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:50:24,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730}] 2024-11-14T21:50:24,167 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:24,167 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,169 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf2 2024-11-14T21:50:24,169 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:24,170 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:24,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621024170Disabling compacts and flushes for region at 1731621024170Disabling writes for close at 1731621024170Writing region close event to WAL at 1731621024173 (+3 ms)Closed at 1731621024173 2024-11-14T21:50:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T21:50:24,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T21:50:24,265 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:24,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-14T21:50:24,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:24,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-14T21:50:24,267 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:24,323 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,323 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:50:24,324 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:24,324 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,324 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,324 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:24,324 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,324 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 264804e76ef9209d727464aa496291f4 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-14T21:50:24,342 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/06d1c2e064bc4a4980ece91724bc6eb9 is 35, key is r1/cf1:q/1731621024129/Put/seqid=0 2024-11-14T21:50:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741887_1065 (size=4783) 2024-11-14T21:50:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741887_1065 (size=4783) 2024-11-14T21:50:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741887_1065 (size=4783) 2024-11-14T21:50:24,353 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/06d1c2e064bc4a4980ece91724bc6eb9 2024-11-14T21:50:24,362 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/06d1c2e064bc4a4980ece91724bc6eb9 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9 2024-11-14T21:50:24,368 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9, entries=1, sequenceid=5, filesize=4.7 K 2024-11-14T21:50:24,370 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 264804e76ef9209d727464aa496291f4 in 45ms, sequenceid=5, compaction requested=false 2024-11-14T21:50:24,370 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-14T21:50:24,377 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T21:50:24,381 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,381 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621024324Running coprocessor pre-close hooks at 1731621024324Disabling compacts and flushes for region at 1731621024324Disabling writes for close at 1731621024324Obtaining lock to block concurrent updates at 1731621024324Preparing flush snapshotting stores in 264804e76ef9209d727464aa496291f4 at 1731621024324Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731621024324Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. at 1731621024325 (+1 ms)Flushing 264804e76ef9209d727464aa496291f4/cf1: creating writer at 1731621024325Flushing 264804e76ef9209d727464aa496291f4/cf1: appending metadata at 1731621024342 (+17 ms)Flushing 264804e76ef9209d727464aa496291f4/cf1: closing flushed file at 1731621024342Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40168566: reopening flushed file at 1731621024361 (+19 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 264804e76ef9209d727464aa496291f4 in 45ms, sequenceid=5, compaction requested=false at 1731621024370 (+9 ms)Writing region close event to WAL at 1731621024373 (+3 ms)Running coprocessor post-close hooks at 1731621024378 (+5 ms)Closed at 1731621024381 (+3 ms) 2024-11-14T21:50:24,382 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 264804e76ef9209d727464aa496291f4 move to ef788a4cade8,39395,1731621004622 record at close sequenceid=5 2024-11-14T21:50:24,386 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=CLOSED 2024-11-14T21:50:24,387 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730 because future has completed 2024-11-14T21:50:24,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T21:50:24,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730 in 228 msec 2024-11-14T21:50:24,395 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE; state=CLOSED, location=ef788a4cade8,39395,1731621004622; forceNewPlan=false, retain=false 2024-11-14T21:50:24,545 INFO [ef788a4cade8:37243 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T21:50:24,546 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPENING, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:24,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE because future has completed 2024-11-14T21:50:24,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622}] 2024-11-14T21:50:24,707 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,708 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:24,708 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,708 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:24,708 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,708 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,710 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,711 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf1 2024-11-14T21:50:24,711 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:24,718 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9 2024-11-14T21:50:24,719 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:24,719 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,720 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf2 2024-11-14T21:50:24,720 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:24,720 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:24,721 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,721 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,723 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,723 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,723 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,726 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T21:50:24,728 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,729 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 264804e76ef9209d727464aa496291f4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59344048, jitterRate=-0.11570477485656738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T21:50:24,730 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:24,730 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 264804e76ef9209d727464aa496291f4: Running coprocessor pre-open hook at 1731621024709Writing region info on filesystem at 1731621024709Initializing all the Stores at 1731621024710 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621024710Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621024710Cleaning up temporary data from old regions at 1731621024723 (+13 ms)Running coprocessor post-open hooks at 1731621024730 (+7 ms)Region opened successfully at 1731621024730 2024-11-14T21:50:24,732 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., pid=9, masterSystemTime=1731621024702 2024-11-14T21:50:24,735 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,735 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:24,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPEN, openSeqNum=9, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:24,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622 because future has completed 2024-11-14T21:50:24,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T21:50:24,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622 in 196 msec 2024-11-14T21:50:24,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE in 596 msec 2024-11-14T21:50:24,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T21:50:24,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T21:50:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:44726 deadline: 1731621084771, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=39395 startCode=1731621004622. As of locationSeqNum=5. 2024-11-14T21:50:24,781 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=39395 startCode=1731621004622. As of locationSeqNum=5. 2024-11-14T21:50:24,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=39395 startCode=1731621004622. As of locationSeqNum=5. 2024-11-14T21:50:24,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,39395,1731621004622, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=39395 startCode=1731621004622. As of locationSeqNum=5. 2024-11-14T21:50:24,892 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:50:24,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:50:24,906 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 264804e76ef9209d727464aa496291f4 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-14T21:50:24,928 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/c511a9d401e241239cf3bfaa1bac2f87 is 29, key is r1/cf1:/1731621024896/DeleteFamily/seqid=0 2024-11-14T21:50:24,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741888_1066 (size=4906) 2024-11-14T21:50:24,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741888_1066 (size=4906) 2024-11-14T21:50:24,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741888_1066 (size=4906) 2024-11-14T21:50:24,945 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:24,956 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:24,979 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf2/a645a1f684ce465a8995c8d56ff12c33 is 29, key is r1/cf2:/1731621024896/DeleteFamily/seqid=0 2024-11-14T21:50:24,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741889_1067 (size=4906) 2024-11-14T21:50:24,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741889_1067 (size=4906) 2024-11-14T21:50:25,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741889_1067 (size=4906) 2024-11-14T21:50:25,000 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf2/a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,023 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/c511a9d401e241239cf3bfaa1bac2f87 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:25,054 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:25,054 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T21:50:25,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf2/a645a1f684ce465a8995c8d56ff12c33 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,070 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,071 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T21:50:25,075 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 264804e76ef9209d727464aa496291f4 in 169ms, sequenceid=12, compaction requested=false 2024-11-14T21:50:25,075 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 264804e76ef9209d727464aa496291f4: 2024-11-14T21:50:25,079 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T21:50:25,081 DEBUG [Time-limited test {}] regionserver.HStore(1541): 264804e76ef9209d727464aa496291f4/cf1 is initiating major compaction (all files) 2024-11-14T21:50:25,081 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T21:50:25,081 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:25,081 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 264804e76ef9209d727464aa496291f4/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,082 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9, hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87] into tmpdir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp, totalSize=9.5 K 2024-11-14T21:50:25,087 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 06d1c2e064bc4a4980ece91724bc6eb9, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731621024129 2024-11-14T21:50:25,088 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c511a9d401e241239cf3bfaa1bac2f87, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T21:50:25,102 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 264804e76ef9209d727464aa496291f4#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:50:25,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741890_1068 (size=4626) 2024-11-14T21:50:25,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741890_1068 (size=4626) 2024-11-14T21:50:25,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741890_1068 (size=4626) 2024-11-14T21:50:25,129 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf1/171df9771f2e4aa5828f62dce99b418e as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/171df9771f2e4aa5828f62dce99b418e 2024-11-14T21:50:25,154 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 264804e76ef9209d727464aa496291f4/cf1 of 264804e76ef9209d727464aa496291f4 into 171df9771f2e4aa5828f62dce99b418e(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:50:25,154 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 264804e76ef9209d727464aa496291f4: 2024-11-14T21:50:25,154 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-14T21:50:25,154 DEBUG [Time-limited test {}] regionserver.HStore(1541): 264804e76ef9209d727464aa496291f4/cf2 is initiating major compaction (all files) 2024-11-14T21:50:25,154 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T21:50:25,154 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T21:50:25,155 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 264804e76ef9209d727464aa496291f4/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,155 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33] into tmpdir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp, totalSize=4.8 K 2024-11-14T21:50:25,156 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a645a1f684ce465a8995c8d56ff12c33, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T21:50:25,175 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 264804e76ef9209d727464aa496291f4#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T21:50:25,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741891_1069 (size=4592) 2024-11-14T21:50:25,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741891_1069 (size=4592) 2024-11-14T21:50:25,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741891_1069 (size=4592) 2024-11-14T21:50:25,214 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/.tmp/cf2/6f97082acb9a468a8435c06340bdee03 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/6f97082acb9a468a8435c06340bdee03 2024-11-14T21:50:25,229 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 264804e76ef9209d727464aa496291f4/cf2 of 264804e76ef9209d727464aa496291f4 into 6f97082acb9a468a8435c06340bdee03(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:50:25,229 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 264804e76ef9209d727464aa496291f4: 2024-11-14T21:50:25,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=264804e76ef9209d727464aa496291f4, source=ef788a4cade8,39395,1731621004622, destination=ef788a4cade8,36817,1731621004730, warming up region on ef788a4cade8,36817,1731621004730 2024-11-14T21:50:25,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=264804e76ef9209d727464aa496291f4, source=ef788a4cade8,39395,1731621004622, destination=ef788a4cade8,36817,1731621004730, running balancer 2024-11-14T21:50:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE 2024-11-14T21:50:25,243 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE 2024-11-14T21:50:25,244 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=CLOSING, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:25,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE because future has completed 2024-11-14T21:50:25,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(7855): Warmup {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:25,253 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T21:50:25,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622}] 2024-11-14T21:50:25,253 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,255 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf1 2024-11-14T21:50:25,255 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:25,264 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9 2024-11-14T21:50:25,269 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/171df9771f2e4aa5828f62dce99b418e 2024-11-14T21:50:25,275 INFO [StoreFileOpener-264804e76ef9209d727464aa496291f4-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:25,275 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:25,276 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:25,276 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,277 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf2 2024-11-14T21:50:25,277 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:25,285 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/6f97082acb9a468a8435c06340bdee03 2024-11-14T21:50:25,290 INFO [StoreFileOpener-264804e76ef9209d727464aa496291f4-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,290 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,290 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:25,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36817 {}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621025291Disabling compacts and flushes for region at 1731621025291Disabling writes for close at 1731621025291Writing region close event to WAL at 1731621025292 (+1 ms)Closed at 1731621025292 2024-11-14T21:50:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-14T21:50:25,408 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,408 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T21:50:25,408 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:25,408 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,408 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,408 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:25,408 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,409 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9, hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87] to archive 2024-11-14T21:50:25,412 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:50:25,416 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9 to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/06d1c2e064bc4a4980ece91724bc6eb9 2024-11-14T21:50:25,418 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87 to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/c511a9d401e241239cf3bfaa1bac2f87 2024-11-14T21:50:25,434 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33] to archive 2024-11-14T21:50:25,435 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:50:25,438 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33 to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/a645a1f684ce465a8995c8d56ff12c33 2024-11-14T21:50:25,445 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-14T21:50:25,446 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,446 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621025408Running coprocessor pre-close hooks at 1731621025408Disabling compacts and flushes for region at 1731621025408Disabling writes for close at 1731621025408Writing region close event to WAL at 1731621025440 (+32 ms)Running coprocessor post-close hooks at 1731621025446 (+6 ms)Closed at 1731621025446 2024-11-14T21:50:25,446 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 264804e76ef9209d727464aa496291f4 move to ef788a4cade8,36817,1731621004730 record at close sequenceid=12 2024-11-14T21:50:25,449 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,450 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=CLOSED 2024-11-14T21:50:25,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622 because future has completed 2024-11-14T21:50:25,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-14T21:50:25,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622 in 210 msec 2024-11-14T21:50:25,468 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE; state=CLOSED, location=ef788a4cade8,36817,1731621004730; forceNewPlan=false, retain=false 2024-11-14T21:50:25,618 INFO [ef788a4cade8:37243 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T21:50:25,619 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPENING, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:25,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE because future has completed 2024-11-14T21:50:25,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730}] 2024-11-14T21:50:25,781 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,781 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:25,782 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,782 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:25,782 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,782 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,784 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,785 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf1 2024-11-14T21:50:25,785 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:25,793 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/171df9771f2e4aa5828f62dce99b418e 2024-11-14T21:50:25,793 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:25,793 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,794 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf2 2024-11-14T21:50:25,794 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:25,803 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/6f97082acb9a468a8435c06340bdee03 2024-11-14T21:50:25,803 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:25,803 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,804 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,806 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,806 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,806 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,807 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T21:50:25,809 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,810 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 264804e76ef9209d727464aa496291f4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69680862, jitterRate=0.038325756788253784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T21:50:25,810 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,811 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 264804e76ef9209d727464aa496291f4: Running coprocessor pre-open hook at 1731621025783Writing region info on filesystem at 1731621025783Initializing all the Stores at 1731621025784 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621025784Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621025784Cleaning up temporary data from old regions at 1731621025806 (+22 ms)Running coprocessor post-open hooks at 1731621025810 (+4 ms)Region opened successfully at 1731621025811 (+1 ms) 2024-11-14T21:50:25,812 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., pid=12, masterSystemTime=1731621025776 2024-11-14T21:50:25,816 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,816 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPEN, openSeqNum=18, regionLocation=ef788a4cade8,36817,1731621004730 2024-11-14T21:50:25,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730 because future has completed 2024-11-14T21:50:25,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-14T21:50:25,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,36817,1731621004730 in 199 msec 2024-11-14T21:50:25,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, REOPEN/MOVE in 584 msec 2024-11-14T21:50:25,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T21:50:25,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T21:50:25,849 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ef788a4cade8,36817,1731621004730: testing ***** 2024-11-14T21:50:25,849 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T21:50:25,852 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T21:50:25,853 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T21:50:25,859 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T21:50:25,860 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T21:50:25,874 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 240436224 }, "NonHeapMemoryUsage": { "committed": 171573248, "init": 7667712, "max": -1, "used": 168953144 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ef788a4cade8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 46, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 18, "ProcessCallTime_98th_percentile": 46, "ProcessCallTime_99th_percentile": 46, "ProcessCallTime_99.9th_percentile": 46, "ProcessCallTime_TimeRangeCount_0-1": 10, "ProcessCallTime_TimeRangeCount_1-3": 11, "ProcessCallTime_TimeRangeCount_3-10": 8, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 5, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 3, "QueueCallTime_98th_percentile": 5, "QueueCallTime_99th_percentile": 5, "QueueCallTime_99.9th_percentile": 5, "QueueCallTime_TimeRangeCount_0-1": 28, "QueueCallTime_TimeRangeCount_1-3": 2, "QueueCallTime_TimeRangeCount_3-10": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 51, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 5, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 18, "TotalCallTime_98th_percentile": 51, "TotalCallTime_99th_percentile": 51, "TotalCallTime_99.9th_percentile": 51, "TotalCallTime_TimeRangeCount_0-1": 8, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 10, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ef788a4cade8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60020", "tag.serverName": "ef788a4cade8,36817,1731621004730", "tag.clusterId": "5db27ac6-dee0-4c18-b507-d6be882807e2", "tag.Context": "regionserver", "tag.Hostname": "ef788a4cade8", "regionCount": 2, "storeCount": 4, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731621004730, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 1, "Put_max": 8, "Put_mean": 2, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 3, "Put_90th_percentile": 5, "Put_95th_percentile": 8, "Put_98th_percentile": 8, "Put_99th_percentile": 8, "Put_99.9th_percentile": 8, "Put_TimeRangeCount_0-1": 4, "Put_TimeRangeCount_1-3": 8, "Put_TimeRangeCount_3-10": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 45, "FlushTime_max": 45, "FlushTime_mean": 45, "FlushTime_25th_percentile": 45, "FlushTime_median": 45, "FlushTime_75th_percentile": 45, "FlushTime_90th_percentile": 45, "FlushTime_95th_percentile": 45, "FlushTime_98th_percentile": 45, "FlushTime_99th_percentile": 45, "FlushTime_99.9th_percentile": 45, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T21:50:25,878 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37243 {}] master.MasterRpcServices(700): ef788a4cade8,36817,1731621004730 reported a fatal error: ***** ABORTING region server ef788a4cade8,36817,1731621004730: testing ***** 2024-11-14T21:50:25,880 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef788a4cade8,36817,1731621004730' ***** 2024-11-14T21:50:25,880 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-14T21:50:25,881 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T21:50:25,881 INFO [RS:1;ef788a4cade8:36817 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T21:50:25,882 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T21:50:25,882 INFO [RS:1;ef788a4cade8:36817 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T21:50:25,882 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(3091): Received CLOSE for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,883 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(956): aborting server ef788a4cade8,36817,1731621004730 2024-11-14T21:50:25,883 INFO [RS:1;ef788a4cade8:36817 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T21:50:25,884 INFO [RS:1;ef788a4cade8:36817 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ef788a4cade8:36817. 2024-11-14T21:50:25,884 DEBUG [RS:1;ef788a4cade8:36817 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:50:25,884 DEBUG [RS:1;ef788a4cade8:36817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:25,885 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T21:50:25,885 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T21:50:25,885 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T21:50:25,885 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T21:50:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39395 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:54426 deadline: 1731621085884, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=36817 startCode=1731621004730. As of locationSeqNum=12. 2024-11-14T21:50:25,887 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:25,887 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,887 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,887 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:25,887 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,887 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T21:50:25,887 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1325): Online Regions={264804e76ef9209d727464aa496291f4=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T21:50:25,888 DEBUG [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:25,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,39395,1731621004622, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,39395,1731621004622, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=36817 startCode=1731621004730. As of locationSeqNum=12. 2024-11-14T21:50:25,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,39395,1731621004622, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=36817 startCode=1731621004730. As of locationSeqNum=12. 2024-11-14T21:50:25,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,39395,1731621004622, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ef788a4cade8 port=36817 startCode=1731621004730. As of locationSeqNum=12. 2024-11-14T21:50:25,888 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T21:50:25,888 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T21:50:25,888 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T21:50:25,888 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T21:50:25,888 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T21:50:25,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ef788a4cade8,36817,1731621004730 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:25,890 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ef788a4cade8,36817,1731621004730 aborting 2024-11-14T21:50:25,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ef788a4cade8,36817,1731621004730 aborting 2024-11-14T21:50:25,891 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 from cache 2024-11-14T21:50:25,899 ERROR [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-14T21:50:25,899 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T21:50:25,900 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T21:50:25,900 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731621025888Running coprocessor pre-close hooks at 1731621025888Disabling compacts and flushes for region at 1731621025888Disabling writes for close at 1731621025888Writing region close event to WAL at 1731621025899 (+11 ms)Running coprocessor post-close hooks at 1731621025899Closed at 1731621025900 (+1 ms) 2024-11-14T21:50:25,900 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T21:50:25,902 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,902 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621025886Running coprocessor pre-close hooks at 1731621025887 (+1 ms)Disabling compacts and flushes for region at 1731621025887Disabling writes for close at 1731621025887Writing region close event to WAL at 1731621025902 (+15 ms)Running coprocessor post-close hooks at 1731621025902Closed at 1731621025902 2024-11-14T21:50:25,903 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:25,931 INFO [regionserver/ef788a4cade8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:50:25,950 INFO [regionserver/ef788a4cade8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T21:50:25,950 INFO [regionserver/ef788a4cade8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T21:50:26,001 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:26,003 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1] 2024-11-14T21:50:26,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ef788a4cade8,36817,1731621004730 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:26,005 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ef788a4cade8,36817,1731621004730 aborting 2024-11-14T21:50:26,005 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ef788a4cade8,36817,1731621004730 aborting 2024-11-14T21:50:26,005 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 from cache 2024-11-14T21:50:26,088 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(976): stopping server ef788a4cade8,36817,1731621004730; all regions closed. 2024-11-14T21:50:26,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741836_1012 (size=6845) 2024-11-14T21:50:26,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741836_1012 (size=6845) 2024-11-14T21:50:26,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741836_1012 (size=6845) 2024-11-14T21:50:26,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741834_1010 (size=2142) 2024-11-14T21:50:26,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741834_1010 (size=2142) 2024-11-14T21:50:26,103 DEBUG [RS:1;ef788a4cade8:36817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:26,103 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:50:26,103 INFO [RS:1;ef788a4cade8:36817 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T21:50:26,103 INFO [RS:1;ef788a4cade8:36817 {}] hbase.ChoreService(370): Chore service for: regionserver/ef788a4cade8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T21:50:26,104 INFO [RS:1;ef788a4cade8:36817 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T21:50:26,104 INFO [regionserver/ef788a4cade8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T21:50:26,104 INFO [RS:1;ef788a4cade8:36817 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36817 2024-11-14T21:50:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741834_1010 (size=2142) 2024-11-14T21:50:26,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:50:26,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef788a4cade8,36817,1731621004730 2024-11-14T21:50:26,112 INFO [RS:1;ef788a4cade8:36817 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T21:50:26,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef788a4cade8,36817,1731621004730] 2024-11-14T21:50:26,117 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef788a4cade8,36817,1731621004730 already deleted, retry=false 2024-11-14T21:50:26,117 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ef788a4cade8,36817,1731621004730 on ef788a4cade8,37243,1731621003759 2024-11-14T21:50:26,122 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ef788a4cade8,36817,1731621004730, splitWal=true, meta=true 2024-11-14T21:50:26,125 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for ef788a4cade8,36817,1731621004730 (carryingMeta=true) ef788a4cade8,36817,1731621004730/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@e46ad6c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T21:50:26,125 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ef788a4cade8,36817,1731621004730, splitWal=true, meta=true 2024-11-14T21:50:26,127 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ef788a4cade8,36817,1731621004730, splitWal=true, meta=true, isMeta: true 2024-11-14T21:50:26,129 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting 2024-11-14T21:50:26,131 INFO [PEWorker-1 {}] master.SplitWALManager(105): ef788a4cade8,36817,1731621004730 WAL count=1, meta=true 2024-11-14T21:50:26,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta}] 2024-11-14T21:50:26,144 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:26,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, worker=ef788a4cade8,39395,1731621004622}] 2024-11-14T21:50:26,211 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:26,213 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1] 2024-11-14T21:50:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36817-0x100ac6b2eed0002, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:26,215 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ef788a4cade8:36817 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ef788a4cade8/172.17.0.2:36817 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:26,215 INFO [RS:1;ef788a4cade8:36817 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T21:50:26,215 INFO [RS:1;ef788a4cade8:36817 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef788a4cade8,36817,1731621004730; zookeeper connection closed. 2024-11-14T21:50:26,216 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1, error=java.net.ConnectException: Call to address=ef788a4cade8:36817 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ef788a4cade8/172.17.0.2:36817 2024-11-14T21:50:26,216 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-14T21:50:26,216 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 from cache 2024-11-14T21:50:26,216 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address ef788a4cade8:36817 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ef788a4cade8/172.17.0.2:36817 2024-11-14T21:50:26,216 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@14cc3894 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@14cc3894 2024-11-14T21:50:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39395 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-14T21:50:26,339 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, size=6.7 K (6845bytes) 2024-11-14T21:50:26,339 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta 2024-11-14T21:50:26,342 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta after 3ms 2024-11-14T21:50:26,350 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta took 11ms 2024-11-14T21:50:26,370 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-14T21:50:26,371 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta so closing down 2024-11-14T21:50:26,371 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:26,391 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp 2024-11-14T21:50:26,394 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp 2024-11-14T21:50:26,394 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:26,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741892_1070 (size=6604) 2024-11-14T21:50:26,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741892_1070 (size=6604) 2024-11-14T21:50:26,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741892_1070 (size=6604) 2024-11-14T21:50:26,521 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:26,523 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1] 2024-11-14T21:50:26,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to ef788a4cade8:36817 this server is in the failed servers list 2024-11-14T21:50:26,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ef788a4cade8:36817 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ef788a4cade8:36817 2024-11-14T21:50:26,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ef788a4cade8:36817 2024-11-14T21:50:26,524 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 from cache 2024-11-14T21:50:26,816 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:26,818 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T21:50:26,818 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 460 ms; skipped=1; WAL=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-14T21:50:26,818 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, journal: Splitting hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, size=6.7 K (6845bytes) at 1731621026339Finishing writing output for hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta so closing down at 1731621026371 (+32 ms)Creating recovered edits writer path=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp at 1731621026394 (+23 ms)3 split writer threads finished at 1731621026394Closed recovered edits writer path=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731621026816 (+422 ms)Rename recovered edits hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta.temp to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731621026818 (+2 ms)Processed 16 edits across 1 Regions in 460 ms; skipped=1; WAL=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1731621026818 2024-11-14T21:50:26,818 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta 2024-11-14T21:50:26,820 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-14T21:50:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-14T21:50:26,829 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs 2024-11-14T21:50:26,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-14T21:50:26,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, worker=ef788a4cade8,39395,1731621004622 in 683 msec 2024-11-14T21:50:26,834 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:26,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T21:50:26,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ef788a4cade8%2C36817%2C1731621004730.meta.1731621006812.meta, worker=ef788a4cade8,39395,1731621004622 in 702 msec 2024-11-14T21:50:26,841 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting dir is empty, no logs to split. 2024-11-14T21:50:26,841 INFO [PEWorker-1 {}] master.SplitWALManager(105): ef788a4cade8,36817,1731621004730 WAL count=0, meta=true 2024-11-14T21:50:26,841 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ef788a4cade8,36817,1731621004730 WAL splitting is done? wals=0, meta=true 2024-11-14T21:50:26,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T21:50:26,845 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T21:50:26,846 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T21:50:26,997 DEBUG [ef788a4cade8:37243 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T21:50:26,997 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(204): Hosts are {ef788a4cade8=0} racks are {/default-rack=0} 2024-11-14T21:50:26,997 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T21:50:26,997 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T21:50:26,997 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T21:50:26,997 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T21:50:26,997 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T21:50:26,997 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T21:50:26,998 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T21:50:26,998 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:27,000 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef788a4cade8,39395,1731621004622, state=OPENING 2024-11-14T21:50:27,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:27,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:27,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:27,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:27,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:27,002 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T21:50:27,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:27,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef788a4cade8,39395,1731621004622}] 2024-11-14T21:50:27,031 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:27,033 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1] 2024-11-14T21:50:27,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39395 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:54426 deadline: 1731621087034, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on ef788a4cade8,39395,1731621004622 2024-11-14T21:50:27,035 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on ef788a4cade8,39395,1731621004622 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:50:27,035 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on ef788a4cade8,39395,1731621004622 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T21:50:27,035 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1 from cache 2024-11-14T21:50:27,161 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T21:50:27,161 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:27,162 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T21:50:27,165 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef788a4cade8%2C39395%2C1731621004622.meta, suffix=.meta, logDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622, archiveDir=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs, maxLogs=32 2024-11-14T21:50:27,182 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622/ef788a4cade8%2C39395%2C1731621004622.meta.1731621027165.meta, exclude list is [], retry=0 2024-11-14T21:50:27,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:27,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:27,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:27,200 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,39395,1731621004622/ef788a4cade8%2C39395%2C1731621004622.meta.1731621027165.meta 2024-11-14T21:50:27,201 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:27,201 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:27,202 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T21:50:27,202 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T21:50:27,202 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T21:50:27,202 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T21:50:27,202 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:27,202 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T21:50:27,202 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T21:50:27,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T21:50:27,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T21:50:27,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:27,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:27,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T21:50:27,208 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T21:50:27,208 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:27,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:27,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T21:50:27,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T21:50:27,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:27,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:27,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T21:50:27,212 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T21:50:27,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:27,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T21:50:27,212 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T21:50:27,213 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740 2024-11-14T21:50:27,215 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740 2024-11-14T21:50:27,216 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T21:50:27,221 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T21:50:27,221 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-14T21:50:27,256 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/info/d594863cd6cf4aa6888d1337b9c36b5a is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4./info:regioninfo/1731621025817/Put/seqid=0 2024-11-14T21:50:27,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741894_1072 (size=11177) 2024-11-14T21:50:27,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741894_1072 (size=11177) 2024-11-14T21:50:27,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741894_1072 (size=11177) 2024-11-14T21:50:27,266 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/info/d594863cd6cf4aa6888d1337b9c36b5a 2024-11-14T21:50:27,298 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/ns/5274916d536249bd8985f50ec55ba973 is 43, key is default/ns:d/1731621006970/Put/seqid=0 2024-11-14T21:50:27,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741895_1073 (size=5153) 2024-11-14T21:50:27,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741895_1073 (size=5153) 2024-11-14T21:50:27,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741895_1073 (size=5153) 2024-11-14T21:50:27,314 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/ns/5274916d536249bd8985f50ec55ba973 2024-11-14T21:50:27,342 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/table/608407948d374cd7ba10cc493397f632 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731621023957/Put/seqid=0 2024-11-14T21:50:27,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741896_1074 (size=5431) 2024-11-14T21:50:27,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741896_1074 (size=5431) 2024-11-14T21:50:27,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741896_1074 (size=5431) 2024-11-14T21:50:27,354 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/table/608407948d374cd7ba10cc493397f632 2024-11-14T21:50:27,362 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/info/d594863cd6cf4aa6888d1337b9c36b5a as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/info/d594863cd6cf4aa6888d1337b9c36b5a 2024-11-14T21:50:27,369 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/info/d594863cd6cf4aa6888d1337b9c36b5a, entries=36, sequenceid=18, filesize=10.9 K 2024-11-14T21:50:27,370 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/ns/5274916d536249bd8985f50ec55ba973 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/ns/5274916d536249bd8985f50ec55ba973 2024-11-14T21:50:27,377 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/ns/5274916d536249bd8985f50ec55ba973, entries=2, sequenceid=18, filesize=5.0 K 2024-11-14T21:50:27,378 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/table/608407948d374cd7ba10cc493397f632 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/table/608407948d374cd7ba10cc493397f632 2024-11-14T21:50:27,384 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/table/608407948d374cd7ba10cc493397f632, entries=2, sequenceid=18, filesize=5.3 K 2024-11-14T21:50:27,385 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 164ms, sequenceid=18, compaction requested=false; wal=null 2024-11-14T21:50:27,386 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T21:50:27,387 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T21:50:27,387 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T21:50:27,389 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T21:50:27,390 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T21:50:27,396 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-14T21:50:27,397 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73945470, jitterRate=0.10187336802482605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T21:50:27,397 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T21:50:27,398 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731621027202Writing region info on filesystem at 1731621027202Initializing all the Stores at 1731621027205 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621027205Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621027205Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621027205Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731621027205Obtaining lock to block concurrent updates at 1731621027222 (+17 ms)Preparing flush snapshotting stores in 1588230740 at 1731621027222Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731621027222Flushing stores of hbase:meta,,1.1588230740 at 1731621027222Flushing 1588230740/info: creating writer at 1731621027222Flushing 1588230740/info: appending metadata at 1731621027255 (+33 ms)Flushing 1588230740/info: closing flushed file at 1731621027256 (+1 ms)Flushing 1588230740/ns: creating writer at 1731621027274 (+18 ms)Flushing 1588230740/ns: appending metadata at 1731621027297 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1731621027297Flushing 1588230740/table: creating writer at 1731621027321 (+24 ms)Flushing 1588230740/table: appending metadata at 1731621027341 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731621027342 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2339c1c8: reopening flushed file at 1731621027361 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29877a7d: reopening flushed file at 1731621027369 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@592a7189: reopening flushed file at 1731621027377 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 164ms, sequenceid=18, compaction requested=false; wal=null at 1731621027385 (+8 ms)Cleaning up temporary data from old regions at 1731621027388 (+3 ms)Running coprocessor post-open hooks at 1731621027398 (+10 ms)Region opened successfully at 1731621027398 2024-11-14T21:50:27,402 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731621027156 2024-11-14T21:50:27,406 DEBUG [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T21:50:27,406 INFO [RS_OPEN_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T21:50:27,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:27,408 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37243 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=ef788a4cade8,39395,1731621004622, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T21:50:27,409 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef788a4cade8,39395,1731621004622, state=OPEN 2024-11-14T21:50:27,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:27,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:27,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T21:50:27,413 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:27,413 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:27,413 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:27,413 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T21:50:27,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-14T21:50:27,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef788a4cade8,39395,1731621004622 in 411 msec 2024-11-14T21:50:27,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-14T21:50:27,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 576 msec 2024-11-14T21:50:27,427 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): ef788a4cade8,36817,1731621004730 had 2 regions 2024-11-14T21:50:27,428 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ef788a4cade8,36817,1731621004730, splitWal=true, meta=true, isMeta: false 2024-11-14T21:50:27,431 INFO [PEWorker-2 {}] master.SplitWALManager(105): ef788a4cade8,36817,1731621004730 WAL count=1, meta=false 2024-11-14T21:50:27,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ef788a4cade8%2C36817%2C1731621004730.1731621006126}] 2024-11-14T21:50:27,434 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=ef788a4cade8,36167,1731621004776 2024-11-14T21:50:27,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ef788a4cade8%2C36817%2C1731621004730.1731621006126, worker=ef788a4cade8,36167,1731621004776}] 2024-11-14T21:50:27,590 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T21:50:27,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T21:50:27,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-14T21:50:27,614 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126, size=2.1 K (2142bytes) 2024-11-14T21:50:27,614 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 2024-11-14T21:50:27,615 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 after 1ms 2024-11-14T21:50:27,621 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 took 7ms 2024-11-14T21:50:27,624 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 264804e76ef9209d727464aa496291f4: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-14T21:50:27,624 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 so closing down 2024-11-14T21:50:27,625 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:27,625 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:27,625 INFO [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-14T21:50:27,625 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126, journal: Splitting hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126, size=2.1 K (2142bytes) at 1731621027614Finishing writing output for hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 so closing down at 1731621027624 (+10 ms)3 split writer threads finished at 1731621027625 (+1 ms)Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731621027625 2024-11-14T21:50:27,625 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 2024-11-14T21:50:27,625 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ef788a4cade8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-14T21:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37243 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-14T21:50:27,634 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting/ef788a4cade8%2C36817%2C1731621004730.1731621006126 to hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs 2024-11-14T21:50:27,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-14T21:50:27,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ef788a4cade8%2C36817%2C1731621004730.1731621006126, worker=ef788a4cade8,36167,1731621004776 in 199 msec 2024-11-14T21:50:27,638 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ef788a4cade8,36167,1731621004776 2024-11-14T21:50:27,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-14T21:50:27,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ef788a4cade8%2C36817%2C1731621004730.1731621006126, worker=ef788a4cade8,36167,1731621004776 in 207 msec 2024-11-14T21:50:27,643 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/WALs/ef788a4cade8,36817,1731621004730-splitting dir is empty, no logs to split. 2024-11-14T21:50:27,643 INFO [PEWorker-2 {}] master.SplitWALManager(105): ef788a4cade8,36817,1731621004730 WAL count=0, meta=false 2024-11-14T21:50:27,643 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ef788a4cade8,36817,1731621004730 WAL splitting is done? wals=0, meta=false 2024-11-14T21:50:27,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN}] 2024-11-14T21:50:27,649 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN 2024-11-14T21:50:27,650 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T21:50:27,800 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(204): Hosts are {ef788a4cade8=0} racks are {/default-rack=0} 2024-11-14T21:50:27,800 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T21:50:27,801 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T21:50:27,801 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T21:50:27,801 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T21:50:27,801 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T21:50:27,801 INFO [ef788a4cade8:37243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T21:50:27,801 DEBUG [ef788a4cade8:37243 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T21:50:27,801 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPENING, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:27,803 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ef788a4cade8:36817 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ef788a4cade8/172.17.0.2:36817 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:27,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1, error=java.net.ConnectException: Call to address=ef788a4cade8:36817 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ef788a4cade8/172.17.0.2:36817 2024-11-14T21:50:27,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-14T21:50:27,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,36817,1731621004730, seqNum=-1 from cache 2024-11-14T21:50:27,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address ef788a4cade8:36817 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ef788a4cade8/172.17.0.2:36817 2024-11-14T21:50:27,911 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:27,911 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1] 2024-11-14T21:50:27,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T21:50:27,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42865, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T21:50:27,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN because future has completed 2024-11-14T21:50:27,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622}] 2024-11-14T21:50:28,051 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T21:50:28,053 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef788a4cade8,39395,1731621004622, seqNum=-1] 2024-11-14T21:50:28,073 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:28,074 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 264804e76ef9209d727464aa496291f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:28,074 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,074 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:28,074 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,074 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,076 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,077 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf1 2024-11-14T21:50:28,077 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,085 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf1/171df9771f2e4aa5828f62dce99b418e 2024-11-14T21:50:28,085 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,085 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,086 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 264804e76ef9209d727464aa496291f4 columnFamilyName cf2 2024-11-14T21:50:28,087 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,094 DEBUG [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/cf2/6f97082acb9a468a8435c06340bdee03 2024-11-14T21:50:28,094 INFO [StoreOpener-264804e76ef9209d727464aa496291f4-1 {}] regionserver.HStore(327): Store=264804e76ef9209d727464aa496291f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,094 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,095 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,096 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,097 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,097 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,098 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T21:50:28,099 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,100 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 264804e76ef9209d727464aa496291f4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60030689, jitterRate=-0.10547302663326263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T21:50:28,100 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:28,101 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 264804e76ef9209d727464aa496291f4: Running coprocessor pre-open hook at 1731621028075Writing region info on filesystem at 1731621028075Initializing all the Stores at 1731621028076 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028076Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028076Cleaning up temporary data from old regions at 1731621028097 (+21 ms)Running coprocessor post-open hooks at 1731621028100 (+3 ms)Region opened successfully at 1731621028101 (+1 ms) 2024-11-14T21:50:28,103 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., pid=21, masterSystemTime=1731621028069 2024-11-14T21:50:28,107 DEBUG [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:28,107 INFO [RS_OPEN_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:28,108 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=264804e76ef9209d727464aa496291f4, regionState=OPEN, openSeqNum=18, regionLocation=ef788a4cade8,39395,1731621004622 2024-11-14T21:50:28,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622 because future has completed 2024-11-14T21:50:28,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-14T21:50:28,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 264804e76ef9209d727464aa496291f4, server=ef788a4cade8,39395,1731621004622 in 196 msec 2024-11-14T21:50:28,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-14T21:50:28,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=264804e76ef9209d727464aa496291f4, ASSIGN in 467 msec 2024-11-14T21:50:28,117 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server ef788a4cade8,36817,1731621004730 after splitting done 2024-11-14T21:50:28,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure ef788a4cade8,36817,1731621004730, splitWal=true, meta=true in 1.9990 sec 2024-11-14T21:50:28,172 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to ef788a4cade8:36817 this server is in the failed servers list 2024-11-14T21:50:28,172 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ef788a4cade8:36817 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ef788a4cade8:36817 2024-11-14T21:50:28,172 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ef788a4cade8:36817 2024-11-14T21:50:28,172 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,36817,1731621004730, seqNum=12 from cache 2024-11-14T21:50:28,390 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., hostname=ef788a4cade8,39395,1731621004622, seqNum=18] 2024-11-14T21:50:28,408 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 415), OpenFileDescriptor=1040 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=637 (was 632) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3798 (was 1277) - AvailableMemoryMB LEAK? - 2024-11-14T21:50:28,410 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-14T21:50:28,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1040, MaxFileDescriptor=1048576, SystemLoadAverage=637, ProcessCount=11, AvailableMemoryMB=3777 2024-11-14T21:50:28,426 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-14T21:50:28,448 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:28,450 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:28,451 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:28,454 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-47753591, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-47753591, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:28,469 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-47753591/hregion-47753591.1731621028455, exclude list is [], retry=0 2024-11-14T21:50:28,474 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:28,475 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:28,475 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:28,481 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-47753591/hregion-47753591.1731621028455 2024-11-14T21:50:28,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:28,482 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7b4263ea61833599701e0386adedf640, NAME => 'testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:28,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741898_1076 (size=67) 2024-11-14T21:50:28,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741898_1076 (size=67) 2024-11-14T21:50:28,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741898_1076 (size=67) 2024-11-14T21:50:28,516 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:28,519 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,521 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName a 2024-11-14T21:50:28,521 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,522 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,522 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,524 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName b 2024-11-14T21:50:28,524 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,525 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,525 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,526 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName c 2024-11-14T21:50:28,526 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,527 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,527 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,528 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,528 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,530 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,530 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,530 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:28,532 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,536 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:28,537 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b4263ea61833599701e0386adedf640; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59022394, jitterRate=-0.12049779295921326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:28,538 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b4263ea61833599701e0386adedf640: Writing region info on filesystem at 1731621028516Initializing all the Stores at 1731621028519 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028519Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028519Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028519Cleaning up temporary data from old regions at 1731621028530 (+11 ms)Region opened successfully at 1731621028538 (+8 ms) 2024-11-14T21:50:28,538 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b4263ea61833599701e0386adedf640, disabling compactions & flushes 2024-11-14T21:50:28,538 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:28,538 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:28,538 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. after waiting 0 ms 2024-11-14T21:50:28,538 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:28,539 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:28,539 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b4263ea61833599701e0386adedf640: Waiting for close lock at 1731621028538Disabling compacts and flushes for region at 1731621028538Disabling writes for close at 1731621028538Writing region close event to WAL at 1731621028539 (+1 ms)Closed at 1731621028539 2024-11-14T21:50:28,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741897_1075 (size=93) 2024-11-14T21:50:28,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741897_1075 (size=93) 2024-11-14T21:50:28,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741897_1075 (size=93) 2024-11-14T21:50:28,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:28,550 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-47753591:(num 1731621028455) 2024-11-14T21:50:28,550 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:28,552 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:28,569 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553, exclude list is [], retry=0 2024-11-14T21:50:28,572 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:28,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:28,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:28,578 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 2024-11-14T21:50:28,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:28,578 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b4263ea61833599701e0386adedf640, NAME => 'testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:28,578 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:28,579 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,579 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,583 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,584 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName a 2024-11-14T21:50:28,584 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,584 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,585 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,585 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName b 2024-11-14T21:50:28,586 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,586 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,586 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,587 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName c 2024-11-14T21:50:28,588 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:28,588 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:28,589 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,590 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,594 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,594 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,595 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:28,597 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:28,598 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b4263ea61833599701e0386adedf640; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72719797, jitterRate=0.08360941708087921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:28,599 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b4263ea61833599701e0386adedf640: Writing region info on filesystem at 1731621028579Initializing all the Stores at 1731621028580 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028580Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028582 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621028582Cleaning up temporary data from old regions at 1731621028594 (+12 ms)Region opened successfully at 1731621028598 (+4 ms) 2024-11-14T21:50:28,657 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b4263ea61833599701e0386adedf640 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T21:50:28,684 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/a/6abdcefe63904d829ea19d4250e38708 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731621028599/Put/seqid=0 2024-11-14T21:50:28,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741900_1078 (size=5958) 2024-11-14T21:50:28,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741900_1078 (size=5958) 2024-11-14T21:50:28,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741900_1078 (size=5958) 2024-11-14T21:50:28,707 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/a/6abdcefe63904d829ea19d4250e38708 2024-11-14T21:50:28,764 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/b/045fe99e5cd04729aa636769f77c330e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731621028619/Put/seqid=0 2024-11-14T21:50:28,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741901_1079 (size=5958) 2024-11-14T21:50:28,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741901_1079 (size=5958) 2024-11-14T21:50:28,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741901_1079 (size=5958) 2024-11-14T21:50:28,823 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/b/045fe99e5cd04729aa636769f77c330e 2024-11-14T21:50:28,876 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/c/fd5278b8ebaf48d09c7f378f36638a04 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731621028642/Put/seqid=0 2024-11-14T21:50:28,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741902_1080 (size=5958) 2024-11-14T21:50:28,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741902_1080 (size=5958) 2024-11-14T21:50:28,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741902_1080 (size=5958) 2024-11-14T21:50:29,322 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/c/fd5278b8ebaf48d09c7f378f36638a04 2024-11-14T21:50:29,330 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/a/6abdcefe63904d829ea19d4250e38708 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/a/6abdcefe63904d829ea19d4250e38708 2024-11-14T21:50:29,338 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/a/6abdcefe63904d829ea19d4250e38708, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T21:50:29,339 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/b/045fe99e5cd04729aa636769f77c330e as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/b/045fe99e5cd04729aa636769f77c330e 2024-11-14T21:50:29,346 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/b/045fe99e5cd04729aa636769f77c330e, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T21:50:29,347 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/c/fd5278b8ebaf48d09c7f378f36638a04 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/c/fd5278b8ebaf48d09c7f378f36638a04 2024-11-14T21:50:29,355 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/c/fd5278b8ebaf48d09c7f378f36638a04, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T21:50:29,356 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 7b4263ea61833599701e0386adedf640 in 700ms, sequenceid=33, compaction requested=false 2024-11-14T21:50:29,356 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7b4263ea61833599701e0386adedf640: 2024-11-14T21:50:29,357 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b4263ea61833599701e0386adedf640, disabling compactions & flushes 2024-11-14T21:50:29,357 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:29,357 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:29,357 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. after waiting 0 ms 2024-11-14T21:50:29,357 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:29,378 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. 2024-11-14T21:50:29,379 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b4263ea61833599701e0386adedf640: Waiting for close lock at 1731621029356Disabling compacts and flushes for region at 1731621029356Disabling writes for close at 1731621029357 (+1 ms)Writing region close event to WAL at 1731621029378 (+21 ms)Closed at 1731621029378 2024-11-14T21:50:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741899_1077 (size=5937) 2024-11-14T21:50:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741899_1077 (size=5937) 2024-11-14T21:50:29,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741899_1077 (size=5937) 2024-11-14T21:50:29,406 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/b/045fe99e5cd04729aa636769f77c330e to hdfs://localhost:45311/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/b/045fe99e5cd04729aa636769f77c330e 2024-11-14T21:50:29,418 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:50:29,430 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553, size=5.8 K (5937bytes) 2024-11-14T21:50:29,430 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 2024-11-14T21:50:29,431 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 after 1ms 2024-11-14T21:50:29,439 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 took 10ms 2024-11-14T21:50:29,462 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731621028553.temp 2024-11-14T21:50:29,466 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 so closing down 2024-11-14T21:50:29,466 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:29,478 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000003-wal.1731621028553.temp 2024-11-14T21:50:29,479 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:29,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741903_1081 (size=5223) 2024-11-14T21:50:29,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741903_1081 (size=5223) 2024-11-14T21:50:29,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741903_1081 (size=5223) 2024-11-14T21:50:29,550 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000003-wal.1731621028553.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:29,558 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000003-wal.1731621028553.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000032 2024-11-14T21:50:29,558 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 108 ms; skipped=2; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-14T21:50:29,558 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553, journal: Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553, size=5.8 K (5937bytes) at 1731621029430Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 so closing down at 1731621029466 (+36 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000003-wal.1731621028553.temp at 1731621029479 (+13 ms)3 split writer threads finished at 1731621029479Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000003-wal.1731621028553.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731621029550 (+71 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000003-wal.1731621028553.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000032 at 1731621029558 (+8 ms)Processed 32 edits across 1 Regions in 108 ms; skipped=2; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731621029558 2024-11-14T21:50:29,575 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621028553 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621028553 2024-11-14T21:50:29,581 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000032 2024-11-14T21:50:29,581 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:29,586 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:29,607 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621029586, exclude list is [], retry=0 2024-11-14T21:50:29,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:29,612 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:29,612 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:29,632 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621029586 2024-11-14T21:50:29,632 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:29,632 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b4263ea61833599701e0386adedf640, NAME => 'testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:29,632 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:29,633 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,633 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,635 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,636 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName a 2024-11-14T21:50:29,636 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:29,647 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/a/6abdcefe63904d829ea19d4250e38708 2024-11-14T21:50:29,647 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:29,647 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,650 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName b 2024-11-14T21:50:29,651 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:29,651 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:29,652 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,654 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b4263ea61833599701e0386adedf640 columnFamilyName c 2024-11-14T21:50:29,654 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:29,668 DEBUG [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/c/fd5278b8ebaf48d09c7f378f36638a04 2024-11-14T21:50:29,668 INFO [StoreOpener-7b4263ea61833599701e0386adedf640-1 {}] regionserver.HStore(327): Store=7b4263ea61833599701e0386adedf640/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:29,670 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,671 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,681 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,692 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000032 2024-11-14T21:50:29,699 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000032 2024-11-14T21:50:29,699 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b4263ea61833599701e0386adedf640 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T21:50:29,718 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/b/ecb7c8f489e745768c4237c8dca07f15 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731621028619/Put/seqid=0 2024-11-14T21:50:29,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741905_1083 (size=5958) 2024-11-14T21:50:29,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741905_1083 (size=5958) 2024-11-14T21:50:29,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741905_1083 (size=5958) 2024-11-14T21:50:29,734 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/b/ecb7c8f489e745768c4237c8dca07f15 2024-11-14T21:50:29,748 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/.tmp/b/ecb7c8f489e745768c4237c8dca07f15 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/b/ecb7c8f489e745768c4237c8dca07f15 2024-11-14T21:50:29,761 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/b/ecb7c8f489e745768c4237c8dca07f15, entries=10, sequenceid=32, filesize=5.8 K 2024-11-14T21:50:29,761 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7b4263ea61833599701e0386adedf640 in 62ms, sequenceid=32, compaction requested=false; wal=null 2024-11-14T21:50:29,763 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/0000000000000000032 2024-11-14T21:50:29,769 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,769 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,770 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:29,772 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b4263ea61833599701e0386adedf640 2024-11-14T21:50:29,780 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/7b4263ea61833599701e0386adedf640/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-14T21:50:29,782 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b4263ea61833599701e0386adedf640; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73427828, jitterRate=0.09415990114212036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:29,783 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b4263ea61833599701e0386adedf640: Writing region info on filesystem at 1731621029633Initializing all the Stores at 1731621029635 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621029635Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621029635Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621029635Obtaining lock to block concurrent updates at 1731621029699 (+64 ms)Preparing flush snapshotting stores in 7b4263ea61833599701e0386adedf640 at 1731621029699Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731621029699Flushing stores of testReplayEditsWrittenViaHRegion,,1731621028448.7b4263ea61833599701e0386adedf640. at 1731621029699Flushing 7b4263ea61833599701e0386adedf640/b: creating writer at 1731621029699Flushing 7b4263ea61833599701e0386adedf640/b: appending metadata at 1731621029717 (+18 ms)Flushing 7b4263ea61833599701e0386adedf640/b: closing flushed file at 1731621029718 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3582522f: reopening flushed file at 1731621029746 (+28 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7b4263ea61833599701e0386adedf640 in 62ms, sequenceid=32, compaction requested=false; wal=null at 1731621029761 (+15 ms)Cleaning up temporary data from old regions at 1731621029769 (+8 ms)Region opened successfully at 1731621029783 (+14 ms) 2024-11-14T21:50:29,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:48974 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:54700 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:36728 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:34870 [Waiting for operation #58] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:38094 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:48312 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1114 (was 1040) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=637 (was 637), ProcessCount=11 (was 11), AvailableMemoryMB=2575 (was 3777) 2024-11-14T21:50:29,826 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-11-14T21:50:29,845 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1114, MaxFileDescriptor=1048576, SystemLoadAverage=637, ProcessCount=11, AvailableMemoryMB=2553 2024-11-14T21:50:29,845 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-11-14T21:50:29,872 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:29,877 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:29,877 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:29,885 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64156326, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-64156326, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:29,908 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64156326/hregion-64156326.1731621029886, exclude list is [], retry=0 2024-11-14T21:50:29,923 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:29,924 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:29,927 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:29,965 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64156326/hregion-64156326.1731621029886 2024-11-14T21:50:29,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:29,976 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 404809c23bb7ff8b1691d4ffd51bde33, NAME => 'testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:30,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741907_1085 (size=68) 2024-11-14T21:50:30,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741907_1085 (size=68) 2024-11-14T21:50:30,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741907_1085 (size=68) 2024-11-14T21:50:30,028 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:30,030 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,033 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName a 2024-11-14T21:50:30,033 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:30,034 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:30,034 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,036 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName b 2024-11-14T21:50:30,036 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:30,036 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:30,037 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,042 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName c 2024-11-14T21:50:30,042 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:30,043 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:30,043 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,046 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,046 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,046 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:30,048 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,051 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:30,051 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 404809c23bb7ff8b1691d4ffd51bde33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65204259, jitterRate=-0.028380826115608215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:30,052 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 404809c23bb7ff8b1691d4ffd51bde33: Writing region info on filesystem at 1731621030029Initializing all the Stores at 1731621030030 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621030030Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621030030Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621030030Cleaning up temporary data from old regions at 1731621030046 (+16 ms)Region opened successfully at 1731621030052 (+6 ms) 2024-11-14T21:50:30,052 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 404809c23bb7ff8b1691d4ffd51bde33, disabling compactions & flushes 2024-11-14T21:50:30,052 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:30,052 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:30,053 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. after waiting 0 ms 2024-11-14T21:50:30,053 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:30,053 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:30,053 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 404809c23bb7ff8b1691d4ffd51bde33: Waiting for close lock at 1731621030052Disabling compacts and flushes for region at 1731621030052Disabling writes for close at 1731621030053 (+1 ms)Writing region close event to WAL at 1731621030053Closed at 1731621030053 2024-11-14T21:50:30,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741906_1084 (size=93) 2024-11-14T21:50:30,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741906_1084 (size=93) 2024-11-14T21:50:30,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741906_1084 (size=93) 2024-11-14T21:50:30,075 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:30,075 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64156326:(num 1731621029886) 2024-11-14T21:50:30,075 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:30,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:30,097 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080, exclude list is [], retry=0 2024-11-14T21:50:30,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:30,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:30,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:30,110 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 2024-11-14T21:50:30,110 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:30,191 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 404809c23bb7ff8b1691d4ffd51bde33, NAME => 'testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:30,197 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,197 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:30,200 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,201 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,205 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,206 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName a 2024-11-14T21:50:30,207 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:30,209 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:30,210 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,211 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName b 2024-11-14T21:50:30,211 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:30,213 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:30,213 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,215 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName c 2024-11-14T21:50:30,215 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:30,218 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:30,218 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,228 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,229 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:30,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,231 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 404809c23bb7ff8b1691d4ffd51bde33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72670566, jitterRate=0.0828758180141449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:30,231 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:30,232 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 404809c23bb7ff8b1691d4ffd51bde33: Running coprocessor pre-open hook at 1731621030201Writing region info on filesystem at 1731621030201Initializing all the Stores at 1731621030204 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621030204Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621030204Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621030204Cleaning up temporary data from old regions at 1731621030228 (+24 ms)Running coprocessor post-open hooks at 1731621030231 (+3 ms)Region opened successfully at 1731621030232 (+1 ms) 2024-11-14T21:50:30,255 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 404809c23bb7ff8b1691d4ffd51bde33 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-14T21:50:30,259 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:30,429 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T21:50:31,260 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:32,260 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:32,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T21:50:33,261 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:34,262 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:34,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-14T21:50:34,264 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:34,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-14T21:50:34,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:35,263 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:35,700 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:50:36,263 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:37,264 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:38,265 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:39,265 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 404809c23bb7ff8b1691d4ffd51bde33/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:39,267 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 404809c23bb7ff8b1691d4ffd51bde33: 2024-11-14T21:50:39,267 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:39,279 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 404809c23bb7ff8b1691d4ffd51bde33: 2024-11-14T21:50:39,279 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-14T21:50:39,279 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 404809c23bb7ff8b1691d4ffd51bde33, disabling compactions & flushes 2024-11-14T21:50:39,279 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:39,279 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:39,279 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. after waiting 0 ms 2024-11-14T21:50:39,279 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:39,280 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:39,280 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. 2024-11-14T21:50:39,280 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 404809c23bb7ff8b1691d4ffd51bde33: Waiting for close lock at 1731621039279Running coprocessor pre-close hooks at 1731621039279Disabling compacts and flushes for region at 1731621039279Disabling writes for close at 1731621039279Writing region close event to WAL at 1731621039280 (+1 ms)Running coprocessor post-close hooks at 1731621039280Closed at 1731621039280 2024-11-14T21:50:39,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741908_1086 (size=3941) 2024-11-14T21:50:39,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741908_1086 (size=3941) 2024-11-14T21:50:39,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741908_1086 (size=3941) 2024-11-14T21:50:39,300 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080, size=3.8 K (3941bytes) 2024-11-14T21:50:39,300 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 2024-11-14T21:50:39,301 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 after 1ms 2024-11-14T21:50:39,304 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 took 4ms 2024-11-14T21:50:39,307 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 so closing down 2024-11-14T21:50:39,308 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:39,309 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731621030080.temp 2024-11-14T21:50:39,310 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000004-wal.1731621030080.temp 2024-11-14T21:50:39,311 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741909_1087 (size=2983) 2024-11-14T21:50:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741909_1087 (size=2983) 2024-11-14T21:50:39,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741909_1087 (size=2983) 2024-11-14T21:50:39,326 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000004-wal.1731621030080.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:39,327 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000004-wal.1731621030080.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000026 2024-11-14T21:50:39,327 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 21 ms; skipped=3; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-14T21:50:39,327 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080, journal: Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080, size=3.8 K (3941bytes) at 1731621039300Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 so closing down at 1731621039307 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000004-wal.1731621030080.temp at 1731621039310 (+3 ms)3 split writer threads finished at 1731621039311 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000004-wal.1731621030080.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731621039326 (+15 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000004-wal.1731621030080.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000026 at 1731621039327 (+1 ms)Processed 23 edits across 1 Regions in 21 ms; skipped=3; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731621039327 2024-11-14T21:50:39,329 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621030080 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621030080 2024-11-14T21:50:39,330 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000026 2024-11-14T21:50:39,330 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:39,332 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:39,353 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621039333, exclude list is [], retry=0 2024-11-14T21:50:39,358 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:39,359 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:39,360 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:39,364 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621039333 2024-11-14T21:50:39,365 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:39,365 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 404809c23bb7ff8b1691d4ffd51bde33, NAME => 'testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:39,366 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,366 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:39,366 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,366 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,371 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,372 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName a 2024-11-14T21:50:39,372 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:39,373 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:39,373 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,374 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName b 2024-11-14T21:50:39,374 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:39,374 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:39,375 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,375 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 404809c23bb7ff8b1691d4ffd51bde33 columnFamilyName c 2024-11-14T21:50:39,375 DEBUG [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:39,376 INFO [StoreOpener-404809c23bb7ff8b1691d4ffd51bde33-1 {}] regionserver.HStore(327): Store=404809c23bb7ff8b1691d4ffd51bde33/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:39,376 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,377 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,379 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000026 2024-11-14T21:50:39,384 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000026 2024-11-14T21:50:39,385 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 404809c23bb7ff8b1691d4ffd51bde33 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-14T21:50:39,409 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/a/f2488c1014f2456485e28a284bd16c3e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731621039271/Put/seqid=0 2024-11-14T21:50:39,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741911_1089 (size=5523) 2024-11-14T21:50:39,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741911_1089 (size=5523) 2024-11-14T21:50:39,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741911_1089 (size=5523) 2024-11-14T21:50:39,423 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/a/f2488c1014f2456485e28a284bd16c3e 2024-11-14T21:50:39,445 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/b/17f5a1173d824f53aece99b1617efe06 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731621039267/Put/seqid=0 2024-11-14T21:50:39,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741912_1090 (size=5524) 2024-11-14T21:50:39,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741912_1090 (size=5524) 2024-11-14T21:50:39,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741912_1090 (size=5524) 2024-11-14T21:50:39,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/b/17f5a1173d824f53aece99b1617efe06 2024-11-14T21:50:39,479 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/c/e12d24b8b77449b8b824905a31b605aa is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731621039268/Put/seqid=0 2024-11-14T21:50:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741913_1091 (size=5457) 2024-11-14T21:50:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741913_1091 (size=5457) 2024-11-14T21:50:39,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741913_1091 (size=5457) 2024-11-14T21:50:39,491 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/c/e12d24b8b77449b8b824905a31b605aa 2024-11-14T21:50:39,497 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/a/f2488c1014f2456485e28a284bd16c3e as hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/a/f2488c1014f2456485e28a284bd16c3e 2024-11-14T21:50:39,504 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/a/f2488c1014f2456485e28a284bd16c3e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T21:50:39,505 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/b/17f5a1173d824f53aece99b1617efe06 as hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/b/17f5a1173d824f53aece99b1617efe06 2024-11-14T21:50:39,516 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/b/17f5a1173d824f53aece99b1617efe06, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T21:50:39,517 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/.tmp/c/e12d24b8b77449b8b824905a31b605aa as hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/c/e12d24b8b77449b8b824905a31b605aa 2024-11-14T21:50:39,525 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/c/e12d24b8b77449b8b824905a31b605aa, entries=6, sequenceid=26, filesize=5.3 K 2024-11-14T21:50:39,525 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 404809c23bb7ff8b1691d4ffd51bde33 in 140ms, sequenceid=26, compaction requested=false; wal=null 2024-11-14T21:50:39,526 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/0000000000000000026 2024-11-14T21:50:39,527 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,527 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,528 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:39,535 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,540 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsAfterAbortingFlush/404809c23bb7ff8b1691d4ffd51bde33/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-14T21:50:39,541 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 404809c23bb7ff8b1691d4ffd51bde33; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64946532, jitterRate=-0.03222125768661499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:39,541 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 404809c23bb7ff8b1691d4ffd51bde33 2024-11-14T21:50:39,542 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 404809c23bb7ff8b1691d4ffd51bde33: Running coprocessor pre-open hook at 1731621039366Writing region info on filesystem at 1731621039366Initializing all the Stores at 1731621039367 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621039367Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621039370 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621039370Obtaining lock to block concurrent updates at 1731621039385 (+15 ms)Preparing flush snapshotting stores in 404809c23bb7ff8b1691d4ffd51bde33 at 1731621039385Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731621039385Flushing stores of testReplayEditsAfterAbortingFlush,,1731621029872.404809c23bb7ff8b1691d4ffd51bde33. at 1731621039385Flushing 404809c23bb7ff8b1691d4ffd51bde33/a: creating writer at 1731621039385Flushing 404809c23bb7ff8b1691d4ffd51bde33/a: appending metadata at 1731621039408 (+23 ms)Flushing 404809c23bb7ff8b1691d4ffd51bde33/a: closing flushed file at 1731621039408Flushing 404809c23bb7ff8b1691d4ffd51bde33/b: creating writer at 1731621039430 (+22 ms)Flushing 404809c23bb7ff8b1691d4ffd51bde33/b: appending metadata at 1731621039444 (+14 ms)Flushing 404809c23bb7ff8b1691d4ffd51bde33/b: closing flushed file at 1731621039444Flushing 404809c23bb7ff8b1691d4ffd51bde33/c: creating writer at 1731621039461 (+17 ms)Flushing 404809c23bb7ff8b1691d4ffd51bde33/c: appending metadata at 1731621039479 (+18 ms)Flushing 404809c23bb7ff8b1691d4ffd51bde33/c: closing flushed file at 1731621039479Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1be5aa54: reopening flushed file at 1731621039496 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73e08b5c: reopening flushed file at 1731621039504 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@432b46fb: reopening flushed file at 1731621039516 (+12 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 404809c23bb7ff8b1691d4ffd51bde33 in 140ms, sequenceid=26, compaction requested=false; wal=null at 1731621039525 (+9 ms)Cleaning up temporary data from old regions at 1731621039527 (+2 ms)Running coprocessor post-open hooks at 1731621039541 (+14 ms)Region opened successfully at 1731621039542 (+1 ms) 2024-11-14T21:50:39,568 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:49558 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:42494 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:39772 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:42478 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:39812 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:49560 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1114) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=683 (was 637) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2096 (was 2553) 2024-11-14T21:50:39,568 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-14T21:50:39,584 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=683, ProcessCount=11, AvailableMemoryMB=2086 2024-11-14T21:50:39,584 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-14T21:50:39,599 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:39,601 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:39,602 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:39,604 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43899842, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-43899842, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:39,618 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43899842/hregion-43899842.1731621039605, exclude list is [], retry=0 2024-11-14T21:50:39,620 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:39,621 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:39,621 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:39,622 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43899842/hregion-43899842.1731621039605 2024-11-14T21:50:39,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637)] 2024-11-14T21:50:39,623 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ca1c094f3605a8a57ed7b6d16c9c6519, NAME => 'testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741915_1093 (size=61) 2024-11-14T21:50:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741915_1093 (size=61) 2024-11-14T21:50:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741915_1093 (size=61) 2024-11-14T21:50:39,632 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:39,633 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,635 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca1c094f3605a8a57ed7b6d16c9c6519 columnFamilyName a 2024-11-14T21:50:39,635 DEBUG [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:39,636 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(327): Store=ca1c094f3605a8a57ed7b6d16c9c6519/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:39,636 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,636 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,637 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,637 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,637 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,638 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,640 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:39,640 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca1c094f3605a8a57ed7b6d16c9c6519; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65749002, jitterRate=-0.02026352286338806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:39,641 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca1c094f3605a8a57ed7b6d16c9c6519: Writing region info on filesystem at 1731621039632Initializing all the Stores at 1731621039633 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621039633Cleaning up temporary data from old regions at 1731621039637 (+4 ms)Region opened successfully at 1731621039641 (+4 ms) 2024-11-14T21:50:39,641 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca1c094f3605a8a57ed7b6d16c9c6519, disabling compactions & flushes 2024-11-14T21:50:39,641 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,641 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,641 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. after waiting 0 ms 2024-11-14T21:50:39,641 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,641 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,642 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca1c094f3605a8a57ed7b6d16c9c6519: Waiting for close lock at 1731621039641Disabling compacts and flushes for region at 1731621039641Disabling writes for close at 1731621039641Writing region close event to WAL at 1731621039641Closed at 1731621039641 2024-11-14T21:50:39,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741914_1092 (size=93) 2024-11-14T21:50:39,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741914_1092 (size=93) 2024-11-14T21:50:39,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741914_1092 (size=93) 2024-11-14T21:50:39,732 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:39,732 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43899842:(num 1731621039605) 2024-11-14T21:50:39,733 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:39,734 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:39,748 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735, exclude list is [], retry=0 2024-11-14T21:50:39,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:39,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:39,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:39,753 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 2024-11-14T21:50:39,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:39,753 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca1c094f3605a8a57ed7b6d16c9c6519, NAME => 'testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:39,753 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:39,754 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,754 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,755 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,756 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca1c094f3605a8a57ed7b6d16c9c6519 columnFamilyName a 2024-11-14T21:50:39,756 DEBUG [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:39,757 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(327): Store=ca1c094f3605a8a57ed7b6d16c9c6519/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:39,757 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,759 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,759 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,759 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,761 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:39,762 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca1c094f3605a8a57ed7b6d16c9c6519; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71667301, jitterRate=0.06792600452899933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:39,763 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca1c094f3605a8a57ed7b6d16c9c6519: Writing region info on filesystem at 1731621039754Initializing all the Stores at 1731621039755 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621039755Cleaning up temporary data from old regions at 1731621039759 (+4 ms)Region opened successfully at 1731621039763 (+4 ms) 2024-11-14T21:50:39,770 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca1c094f3605a8a57ed7b6d16c9c6519, disabling compactions & flushes 2024-11-14T21:50:39,770 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,771 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,771 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. after waiting 0 ms 2024-11-14T21:50:39,771 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,771 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,771 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. 2024-11-14T21:50:39,771 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca1c094f3605a8a57ed7b6d16c9c6519: Waiting for close lock at 1731621039770Disabling compacts and flushes for region at 1731621039770Disabling writes for close at 1731621039771 (+1 ms)Writing region close event to WAL at 1731621039771Closed at 1731621039771 2024-11-14T21:50:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741916_1094 (size=1623) 2024-11-14T21:50:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741916_1094 (size=1623) 2024-11-14T21:50:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741916_1094 (size=1623) 2024-11-14T21:50:39,798 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735, size=1.6 K (1623bytes) 2024-11-14T21:50:39,798 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 2024-11-14T21:50:39,799 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 after 1ms 2024-11-14T21:50:39,801 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 took 3ms 2024-11-14T21:50:39,803 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 so closing down 2024-11-14T21:50:39,803 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:39,804 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731621039735.temp 2024-11-14T21:50:39,806 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000003-wal.1731621039735.temp 2024-11-14T21:50:39,806 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741917_1095 (size=1623) 2024-11-14T21:50:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741917_1095 (size=1623) 2024-11-14T21:50:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741917_1095 (size=1623) 2024-11-14T21:50:39,813 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000003-wal.1731621039735.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:39,814 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000003-wal.1731621039735.temp to hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000012 2024-11-14T21:50:39,814 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-14T21:50:39,814 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735, journal: Splitting hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735, size=1.6 K (1623bytes) at 1731621039798Finishing writing output for hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 so closing down at 1731621039803 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000003-wal.1731621039735.temp at 1731621039806 (+3 ms)3 split writer threads finished at 1731621039806Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000003-wal.1731621039735.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731621039813 (+7 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000003-wal.1731621039735.temp to hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000012 at 1731621039814 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731621039814 2024-11-14T21:50:39,816 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621039735 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621039735 2024-11-14T21:50:39,817 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000012 2024-11-14T21:50:40,138 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:40,196 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:40,210 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621040196, exclude list is [], retry=0 2024-11-14T21:50:40,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:40,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:40,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:40,230 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621040196 2024-11-14T21:50:40,231 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca1c094f3605a8a57ed7b6d16c9c6519, NAME => 'testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,234 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,236 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca1c094f3605a8a57ed7b6d16c9c6519 columnFamilyName a 2024-11-14T21:50:40,236 DEBUG [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,237 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(327): Store=ca1c094f3605a8a57ed7b6d16c9c6519/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,237 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,240 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000012 2024-11-14T21:50:40,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000012 2024-11-14T21:50:40,243 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca1c094f3605a8a57ed7b6d16c9c6519 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-14T21:50:40,260 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/.tmp/a/6b24c45c4cfa455c91203cfca4d36f24 is 79, key is testDatalossWhenInputError/a:x0/1731621039763/Put/seqid=0 2024-11-14T21:50:40,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741919_1097 (size=5808) 2024-11-14T21:50:40,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741919_1097 (size=5808) 2024-11-14T21:50:40,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741919_1097 (size=5808) 2024-11-14T21:50:40,267 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/.tmp/a/6b24c45c4cfa455c91203cfca4d36f24 2024-11-14T21:50:40,276 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/.tmp/a/6b24c45c4cfa455c91203cfca4d36f24 as hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/a/6b24c45c4cfa455c91203cfca4d36f24 2024-11-14T21:50:40,284 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/a/6b24c45c4cfa455c91203cfca4d36f24, entries=10, sequenceid=12, filesize=5.7 K 2024-11-14T21:50:40,284 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ca1c094f3605a8a57ed7b6d16c9c6519 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-11-14T21:50:40,285 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/0000000000000000012 2024-11-14T21:50:40,286 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,286 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,290 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-14T21:50:40,291 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca1c094f3605a8a57ed7b6d16c9c6519; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74288245, jitterRate=0.10698111355304718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:40,292 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca1c094f3605a8a57ed7b6d16c9c6519: Writing region info on filesystem at 1731621040231Initializing all the Stores at 1731621040234 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040234Obtaining lock to block concurrent updates at 1731621040243 (+9 ms)Preparing flush snapshotting stores in ca1c094f3605a8a57ed7b6d16c9c6519 at 1731621040243Finished memstore snapshotting testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731621040243Flushing stores of testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519. at 1731621040243Flushing ca1c094f3605a8a57ed7b6d16c9c6519/a: creating writer at 1731621040243Flushing ca1c094f3605a8a57ed7b6d16c9c6519/a: appending metadata at 1731621040259 (+16 ms)Flushing ca1c094f3605a8a57ed7b6d16c9c6519/a: closing flushed file at 1731621040259Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d9520c7: reopening flushed file at 1731621040275 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ca1c094f3605a8a57ed7b6d16c9c6519 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1731621040284 (+9 ms)Cleaning up temporary data from old regions at 1731621040286 (+2 ms)Region opened successfully at 1731621040292 (+6 ms) 2024-11-14T21:50:40,296 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca1c094f3605a8a57ed7b6d16c9c6519, NAME => 'testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:40,296 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731621039600.ca1c094f3605a8a57ed7b6d16c9c6519.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:40,296 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,296 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,298 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,298 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca1c094f3605a8a57ed7b6d16c9c6519 columnFamilyName a 2024-11-14T21:50:40,298 DEBUG [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,304 DEBUG [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/a/6b24c45c4cfa455c91203cfca4d36f24 2024-11-14T21:50:40,304 INFO [StoreOpener-ca1c094f3605a8a57ed7b6d16c9c6519-1 {}] regionserver.HStore(327): Store=ca1c094f3605a8a57ed7b6d16c9c6519/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,308 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca1c094f3605a8a57ed7b6d16c9c6519 2024-11-14T21:50:40,310 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testDatalossWhenInputError/ca1c094f3605a8a57ed7b6d16c9c6519/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-14T21:50:40,311 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca1c094f3605a8a57ed7b6d16c9c6519; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73324555, jitterRate=0.09262101352214813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T21:50:40,311 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca1c094f3605a8a57ed7b6d16c9c6519: Writing region info on filesystem at 1731621040296Initializing all the Stores at 1731621040297 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040297Cleaning up temporary data from old regions at 1731621040306 (+9 ms)Region opened successfully at 1731621040311 (+5 ms) 2024-11-14T21:50:40,328 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:42548 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:39848 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:39812 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:49560 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2030617010-172.17.0.2-1731621000517:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:49642 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1266 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=700 (was 683) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2079 (was 2086) 2024-11-14T21:50:40,328 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-14T21:50:40,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1266, MaxFileDescriptor=1048576, SystemLoadAverage=700, ProcessCount=11, AvailableMemoryMB=2077 2024-11-14T21:50:40,342 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-14T21:50:40,357 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:40,359 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:40,359 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:40,361 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35545746, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-35545746, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:40,374 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35545746/hregion-35545746.1731621040362, exclude list is [], retry=0 2024-11-14T21:50:40,377 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:40,377 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:40,378 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:40,382 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35545746/hregion-35545746.1731621040362 2024-11-14T21:50:40,382 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:40,382 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 58c4a65e4f7af5495ed67d3d3bd958ec, NAME => 'testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:40,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741921_1099 (size=63) 2024-11-14T21:50:40,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741921_1099 (size=63) 2024-11-14T21:50:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741921_1099 (size=63) 2024-11-14T21:50:40,476 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:40,477 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,479 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName a 2024-11-14T21:50:40,479 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,479 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,480 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,481 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName b 2024-11-14T21:50:40,481 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,481 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,481 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,482 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName c 2024-11-14T21:50:40,482 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,483 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,483 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,484 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,485 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,485 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,485 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:40,487 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,489 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:40,489 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 58c4a65e4f7af5495ed67d3d3bd958ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60883153, jitterRate=-0.09277032315731049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:40,489 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 58c4a65e4f7af5495ed67d3d3bd958ec: Writing region info on filesystem at 1731621040476Initializing all the Stores at 1731621040477 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040477Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040477Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040477Cleaning up temporary data from old regions at 1731621040485 (+8 ms)Region opened successfully at 1731621040489 (+4 ms) 2024-11-14T21:50:40,490 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 58c4a65e4f7af5495ed67d3d3bd958ec, disabling compactions & flushes 2024-11-14T21:50:40,490 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:40,490 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:40,490 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. after waiting 0 ms 2024-11-14T21:50:40,490 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:40,490 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:40,490 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 58c4a65e4f7af5495ed67d3d3bd958ec: Waiting for close lock at 1731621040490Disabling compacts and flushes for region at 1731621040490Disabling writes for close at 1731621040490Writing region close event to WAL at 1731621040490Closed at 1731621040490 2024-11-14T21:50:40,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741920_1098 (size=93) 2024-11-14T21:50:40,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741920_1098 (size=93) 2024-11-14T21:50:40,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741920_1098 (size=93) 2024-11-14T21:50:40,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:40,497 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35545746:(num 1731621040362) 2024-11-14T21:50:40,497 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:40,499 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:40,513 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499, exclude list is [], retry=0 2024-11-14T21:50:40,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:40,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:40,516 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:40,517 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 2024-11-14T21:50:40,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:40,517 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c4a65e4f7af5495ed67d3d3bd958ec, NAME => 'testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:40,518 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:40,518 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,518 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,521 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,522 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName a 2024-11-14T21:50:40,522 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,523 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,523 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,523 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName b 2024-11-14T21:50:40,523 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,524 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,524 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,524 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName c 2024-11-14T21:50:40,524 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:40,525 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:40,525 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,527 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,527 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,528 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:40,529 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:40,530 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 58c4a65e4f7af5495ed67d3d3bd958ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74296187, jitterRate=0.10709945857524872}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:40,531 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 58c4a65e4f7af5495ed67d3d3bd958ec: Writing region info on filesystem at 1731621040518Initializing all the Stores at 1731621040519 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040519Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040521 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621040521Cleaning up temporary data from old regions at 1731621040527 (+6 ms)Region opened successfully at 1731621040531 (+4 ms) 2024-11-14T21:50:40,639 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731621040639/Put/seqid=0 2024-11-14T21:50:40,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741923_1101 (size=4875) 2024-11-14T21:50:40,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741923_1101 (size=4875) 2024-11-14T21:50:40,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741923_1101 (size=4875) 2024-11-14T21:50:40,651 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731621040651/Put/seqid=0 2024-11-14T21:50:40,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741924_1102 (size=4875) 2024-11-14T21:50:40,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741924_1102 (size=4875) 2024-11-14T21:50:40,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741924_1102 (size=4875) 2024-11-14T21:50:40,662 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731621040662/Put/seqid=0 2024-11-14T21:50:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741925_1103 (size=4875) 2024-11-14T21:50:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741925_1103 (size=4875) 2024-11-14T21:50:40,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741925_1103 (size=4875) 2024-11-14T21:50:41,069 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 58c4a65e4f7af5495ed67d3d3bd958ec/a 2024-11-14T21:50:41,073 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-14T21:50:41,073 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T21:50:41,073 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 58c4a65e4f7af5495ed67d3d3bd958ec/a 2024-11-14T21:50:41,076 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-14T21:50:41,076 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T21:50:41,076 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 58c4a65e4f7af5495ed67d3d3bd958ec/a 2024-11-14T21:50:41,079 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-14T21:50:41,079 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T21:50:41,079 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 58c4a65e4f7af5495ed67d3d3bd958ec 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-14T21:50:41,095 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/.tmp/a/d1eabb8154d14fc9a5fa879da786440d is 55, key is testCompactedBulkLoadedFiles/a:a/1731621040531/Put/seqid=0 2024-11-14T21:50:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741926_1104 (size=5107) 2024-11-14T21:50:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741926_1104 (size=5107) 2024-11-14T21:50:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741926_1104 (size=5107) 2024-11-14T21:50:41,105 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/.tmp/a/d1eabb8154d14fc9a5fa879da786440d 2024-11-14T21:50:41,111 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/.tmp/a/d1eabb8154d14fc9a5fa879da786440d as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d 2024-11-14T21:50:41,116 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d, entries=1, sequenceid=4, filesize=5.0 K 2024-11-14T21:50:41,117 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 58c4a65e4f7af5495ed67d3d3bd958ec in 38ms, sequenceid=4, compaction requested=false 2024-11-14T21:50:41,117 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 58c4a65e4f7af5495ed67d3d3bd958ec: 2024-11-14T21:50:41,118 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ 2024-11-14T21:50:41,119 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ 2024-11-14T21:50:41,120 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ 2024-11-14T21:50:41,121 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile0 into 58c4a65e4f7af5495ed67d3d3bd958ec/a as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ - updating store file list. 2024-11-14T21:50:41,125 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e25e03b8e2b04178b38f3b92427388c0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:41,125 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ into 58c4a65e4f7af5495ed67d3d3bd958ec/a 2024-11-14T21:50:41,125 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile0 into 58c4a65e4f7af5495ed67d3d3bd958ec/a (new location: hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_) 2024-11-14T21:50:41,126 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile1 into 58c4a65e4f7af5495ed67d3d3bd958ec/a as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ - updating store file list. 2024-11-14T21:50:41,130 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5514c6c3204f4fa48085f120f47e758b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:41,130 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ into 58c4a65e4f7af5495ed67d3d3bd958ec/a 2024-11-14T21:50:41,130 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile1 into 58c4a65e4f7af5495ed67d3d3bd958ec/a (new location: hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_) 2024-11-14T21:50:41,131 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile2 into 58c4a65e4f7af5495ed67d3d3bd958ec/a as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ - updating store file list. 2024-11-14T21:50:41,135 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:41,136 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ into 58c4a65e4f7af5495ed67d3d3bd958ec/a 2024-11-14T21:50:41,136 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45311/hbase/testCompactedBulkLoadedFiles/hfile2 into 58c4a65e4f7af5495ed67d3d3bd958ec/a (new location: hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_) 2024-11-14T21:50:41,144 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T21:50:41,144 DEBUG [Time-limited test {}] regionserver.HStore(1541): 58c4a65e4f7af5495ed67d3d3bd958ec/a is initiating major compaction (all files) 2024-11-14T21:50:41,144 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 58c4a65e4f7af5495ed67d3d3bd958ec/a in testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:41,144 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_] into tmpdir=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/.tmp, totalSize=19.3 K 2024-11-14T21:50:41,145 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d1eabb8154d14fc9a5fa879da786440d, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731621040531 2024-11-14T21:50:41,145 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e25e03b8e2b04178b38f3b92427388c0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T21:50:41,146 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5514c6c3204f4fa48085f120f47e758b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T21:50:41,146 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T21:50:41,157 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/.tmp/a/37c184afe6ca4d8ba1171d9aafce72b8 is 55, key is testCompactedBulkLoadedFiles/a:a/1731621040531/Put/seqid=0 2024-11-14T21:50:41,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741927_1105 (size=6154) 2024-11-14T21:50:41,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741927_1105 (size=6154) 2024-11-14T21:50:41,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741927_1105 (size=6154) 2024-11-14T21:50:41,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/.tmp/a/37c184afe6ca4d8ba1171d9aafce72b8 as hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/37c184afe6ca4d8ba1171d9aafce72b8 2024-11-14T21:50:41,175 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 58c4a65e4f7af5495ed67d3d3bd958ec/a of 58c4a65e4f7af5495ed67d3d3bd958ec into 37c184afe6ca4d8ba1171d9aafce72b8(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T21:50:41,175 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 58c4a65e4f7af5495ed67d3d3bd958ec: 2024-11-14T21:50:41,175 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T21:50:41,175 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T21:50:41,209 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499, size=0 (0bytes) 2024-11-14T21:50:41,209 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 might be still open, length is 0 2024-11-14T21:50:41,209 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 2024-11-14T21:50:41,209 WARN [IPC Server handler 1 on default port 45311 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-14T21:50:41,209 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 after 0ms 2024-11-14T21:50:41,674 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:49678 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:32913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49678 dst: /127.0.0.1:32913 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:32913 remote=/127.0.0.1:49678]. Total timeout mills is 60000, 59502 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:41,674 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:42580 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39681:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42580 dst: /127.0.0.1:39681 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:41,674 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:39882 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:44645:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39882 dst: /127.0.0.1:44645 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:41,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741922_1106 (size=1659) 2024-11-14T21:50:41,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741922_1106 (size=1659) 2024-11-14T21:50:44,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-14T21:50:44,264 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:44,265 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-14T21:50:44,265 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-14T21:50:44,782 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T21:50:45,210 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 after 4001ms 2024-11-14T21:50:45,213 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 took 4005ms 2024-11-14T21:50:45,215 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499; continuing. 2024-11-14T21:50:45,215 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 so closing down 2024-11-14T21:50:45,215 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:45,217 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731621040499.temp 2024-11-14T21:50:45,218 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000003-wal.1731621040499.temp 2024-11-14T21:50:45,218 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:45,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741928_1107 (size=698) 2024-11-14T21:50:45,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741928_1107 (size=698) 2024-11-14T21:50:45,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741928_1107 (size=698) 2024-11-14T21:50:45,225 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000003-wal.1731621040499.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:45,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000003-wal.1731621040499.temp to hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000008 2024-11-14T21:50:45,227 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T21:50:45,227 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499, journal: Splitting hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499, size=0 (0bytes) at 1731621041209Finishing writing output for hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 so closing down at 1731621045215 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000003-wal.1731621040499.temp at 1731621045218 (+3 ms)3 split writer threads finished at 1731621045219 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000003-wal.1731621040499.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731621045225 (+6 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000003-wal.1731621040499.temp to hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000008 at 1731621045227 (+2 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499, size=0, length=0, corrupted=false, cancelled=false at 1731621045227 2024-11-14T21:50:45,229 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621040499 2024-11-14T21:50:45,229 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000008 2024-11-14T21:50:45,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:45,231 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:45,245 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621045231, exclude list is [], retry=0 2024-11-14T21:50:45,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:45,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:45,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:45,249 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621045231 2024-11-14T21:50:45,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:45,249 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c4a65e4f7af5495ed67d3d3bd958ec, NAME => 'testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:45,250 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:45,250 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,250 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,251 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,252 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName a 2024-11-14T21:50:45,252 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,258 DEBUG [StoreFileOpener-58c4a65e4f7af5495ed67d3d3bd958ec-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:45,258 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ 2024-11-14T21:50:45,261 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/37c184afe6ca4d8ba1171d9aafce72b8 2024-11-14T21:50:45,265 DEBUG [StoreFileOpener-58c4a65e4f7af5495ed67d3d3bd958ec-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5514c6c3204f4fa48085f120f47e758b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:45,265 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ 2024-11-14T21:50:45,269 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d 2024-11-14T21:50:45,272 DEBUG [StoreFileOpener-58c4a65e4f7af5495ed67d3d3bd958ec-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e25e03b8e2b04178b38f3b92427388c0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T21:50:45,272 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ 2024-11-14T21:50:45,273 WARN [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bbb775e 2024-11-14T21:50:45,273 WARN [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bbb775e 2024-11-14T21:50:45,273 WARN [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bbb775e 2024-11-14T21:50:45,273 WARN [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bbb775e 2024-11-14T21:50:45,273 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_] to archive 2024-11-14T21:50:45,274 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T21:50:45,275 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ to hdfs://localhost:45311/hbase/archive/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_ 2024-11-14T21:50:45,276 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ to hdfs://localhost:45311/hbase/archive/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/5514c6c3204f4fa48085f120f47e758b_SeqId_4_ 2024-11-14T21:50:45,277 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d to hdfs://localhost:45311/hbase/archive/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/d1eabb8154d14fc9a5fa879da786440d 2024-11-14T21:50:45,278 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ to hdfs://localhost:45311/hbase/archive/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/a/e25e03b8e2b04178b38f3b92427388c0_SeqId_4_ 2024-11-14T21:50:45,278 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,279 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,279 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName b 2024-11-14T21:50:45,280 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,280 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,280 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,281 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c4a65e4f7af5495ed67d3d3bd958ec columnFamilyName c 2024-11-14T21:50:45,281 DEBUG [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,282 INFO [StoreOpener-58c4a65e4f7af5495ed67d3d3bd958ec-1 {}] regionserver.HStore(327): Store=58c4a65e4f7af5495ed67d3d3bd958ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,282 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,285 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000008 2024-11-14T21:50:45,289 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 58c4a65e4f7af5495ed67d3d3bd958ec : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "58c4a65e4f7af5495ed67d3d3bd958ec" family_name: "a" compaction_input: "d1eabb8154d14fc9a5fa879da786440d" compaction_input: "e25e03b8e2b04178b38f3b92427388c0_SeqId_4_" compaction_input: "5514c6c3204f4fa48085f120f47e758b_SeqId_4_" compaction_input: "264c7bcdde4a48e3a2b8ae2c76dbaaaa_SeqId_4_" compaction_output: "37c184afe6ca4d8ba1171d9aafce72b8" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-14T21:50:45,289 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-14T21:50:45,289 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000008 2024-11-14T21:50:45,290 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/0000000000000000008 2024-11-14T21:50:45,291 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,291 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,291 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:45,293 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 58c4a65e4f7af5495ed67d3d3bd958ec 2024-11-14T21:50:45,299 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testCompactedBulkLoadedFiles/58c4a65e4f7af5495ed67d3d3bd958ec/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T21:50:45,300 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 58c4a65e4f7af5495ed67d3d3bd958ec; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70711369, jitterRate=0.05368150770664215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:45,300 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 58c4a65e4f7af5495ed67d3d3bd958ec: Writing region info on filesystem at 1731621045250Initializing all the Stores at 1731621045251 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045251Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045251Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045251Cleaning up temporary data from old regions at 1731621045291 (+40 ms)Region opened successfully at 1731621045300 (+9 ms) 2024-11-14T21:50:45,302 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 58c4a65e4f7af5495ed67d3d3bd958ec, disabling compactions & flushes 2024-11-14T21:50:45,302 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:45,302 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:45,302 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. after waiting 0 ms 2024-11-14T21:50:45,302 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:45,303 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731621040357.58c4a65e4f7af5495ed67d3d3bd958ec. 2024-11-14T21:50:45,303 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 58c4a65e4f7af5495ed67d3d3bd958ec: Waiting for close lock at 1731621045302Disabling compacts and flushes for region at 1731621045302Disabling writes for close at 1731621045302Writing region close event to WAL at 1731621045303 (+1 ms)Closed at 1731621045303 2024-11-14T21:50:45,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741929_1108 (size=93) 2024-11-14T21:50:45,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741929_1108 (size=93) 2024-11-14T21:50:45,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741929_1108 (size=93) 2024-11-14T21:50:45,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:45,308 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731621045231) 2024-11-14T21:50:45,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=441 (was 430) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45311 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:45311 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1540903337_22 at /127.0.0.1:49754 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1540903337_22 at /127.0.0.1:42666 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1540903337_22 at /127.0.0.1:39958 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1266) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=676 (was 700), ProcessCount=11 (was 11), AvailableMemoryMB=1793 (was 2077) 2024-11-14T21:50:45,326 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-14T21:50:45,341 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=441, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=676, ProcessCount=11, AvailableMemoryMB=1792 2024-11-14T21:50:45,341 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-14T21:50:45,358 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:45,360 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T21:50:45,360 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T21:50:45,362 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16023541, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/hregion-16023541, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:45,375 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16023541/hregion-16023541.1731621045362, exclude list is [], retry=0 2024-11-14T21:50:45,377 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:45,377 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:45,378 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:45,379 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16023541/hregion-16023541.1731621045362 2024-11-14T21:50:45,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:45,380 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4dd8f9bce73d9083aae557fcb2ca7d14, NAME => 'testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45311/hbase 2024-11-14T21:50:45,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741931_1110 (size=67) 2024-11-14T21:50:45,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741931_1110 (size=67) 2024-11-14T21:50:45,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741931_1110 (size=67) 2024-11-14T21:50:45,389 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:45,391 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,392 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName a 2024-11-14T21:50:45,392 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,392 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,393 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,394 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName b 2024-11-14T21:50:45,394 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,395 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,395 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,396 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName c 2024-11-14T21:50:45,396 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,396 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,396 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,397 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,397 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,398 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,398 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,399 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:45,400 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,402 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T21:50:45,403 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4dd8f9bce73d9083aae557fcb2ca7d14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67808435, jitterRate=0.010424420237541199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:45,403 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Writing region info on filesystem at 1731621045389Initializing all the Stores at 1731621045390 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045390Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045390Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045391 (+1 ms)Cleaning up temporary data from old regions at 1731621045398 (+7 ms)Region opened successfully at 1731621045403 (+5 ms) 2024-11-14T21:50:45,403 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4dd8f9bce73d9083aae557fcb2ca7d14, disabling compactions & flushes 2024-11-14T21:50:45,403 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,403 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,403 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. after waiting 0 ms 2024-11-14T21:50:45,403 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,403 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,404 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Waiting for close lock at 1731621045403Disabling compacts and flushes for region at 1731621045403Disabling writes for close at 1731621045403Writing region close event to WAL at 1731621045403Closed at 1731621045403 2024-11-14T21:50:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741930_1109 (size=93) 2024-11-14T21:50:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741930_1109 (size=93) 2024-11-14T21:50:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741930_1109 (size=93) 2024-11-14T21:50:45,408 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:45,408 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16023541:(num 1731621045362) 2024-11-14T21:50:45,409 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:45,410 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:45,431 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411, exclude list is [], retry=0 2024-11-14T21:50:45,434 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:45,434 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:45,435 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:45,436 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 2024-11-14T21:50:45,437 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:45,438 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4dd8f9bce73d9083aae557fcb2ca7d14, NAME => 'testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:45,438 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:45,438 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,438 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,439 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,440 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName a 2024-11-14T21:50:45,440 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,441 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,441 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,442 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName b 2024-11-14T21:50:45,442 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,442 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,442 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,443 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName c 2024-11-14T21:50:45,443 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,443 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,443 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,445 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,446 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,446 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,446 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:45,447 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,448 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4dd8f9bce73d9083aae557fcb2ca7d14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66277526, jitterRate=-0.012387901544570923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:45,448 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Writing region info on filesystem at 1731621045438Initializing all the Stores at 1731621045439 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045439Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045439Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045439Cleaning up temporary data from old regions at 1731621045446 (+7 ms)Region opened successfully at 1731621045448 (+2 ms) 2024-11-14T21:50:45,456 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4dd8f9bce73d9083aae557fcb2ca7d14 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T21:50:45,478 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/a/94c099fa9cc64dc09465e1fd9b5c9669 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731621045448/Put/seqid=0 2024-11-14T21:50:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741933_1112 (size=5958) 2024-11-14T21:50:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741933_1112 (size=5958) 2024-11-14T21:50:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741933_1112 (size=5958) 2024-11-14T21:50:45,486 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/a/94c099fa9cc64dc09465e1fd9b5c9669 2024-11-14T21:50:45,492 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/a/94c099fa9cc64dc09465e1fd9b5c9669 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/a/94c099fa9cc64dc09465e1fd9b5c9669 2024-11-14T21:50:45,497 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/a/94c099fa9cc64dc09465e1fd9b5c9669, entries=10, sequenceid=13, filesize=5.8 K 2024-11-14T21:50:45,498 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4dd8f9bce73d9083aae557fcb2ca7d14 in 42ms, sequenceid=13, compaction requested=false 2024-11-14T21:50:45,498 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4dd8f9bce73d9083aae557fcb2ca7d14: 2024-11-14T21:50:45,513 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4dd8f9bce73d9083aae557fcb2ca7d14, disabling compactions & flushes 2024-11-14T21:50:45,513 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,513 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,513 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. after waiting 0 ms 2024-11-14T21:50:45,513 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,514 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,514 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:45,514 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Waiting for close lock at 1731621045513Disabling compacts and flushes for region at 1731621045513Disabling writes for close at 1731621045513Writing region close event to WAL at 1731621045514 (+1 ms)Closed at 1731621045514 2024-11-14T21:50:45,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741932_1111 (size=5869) 2024-11-14T21:50:45,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741932_1111 (size=5869) 2024-11-14T21:50:45,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741932_1111 (size=5869) 2024-11-14T21:50:45,533 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411, size=5.7 K (5869bytes) 2024-11-14T21:50:45,533 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 2024-11-14T21:50:45,533 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 after 0ms 2024-11-14T21:50:45,536 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 took 3ms 2024-11-14T21:50:45,537 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 so closing down 2024-11-14T21:50:45,537 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:45,539 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731621045411.temp 2024-11-14T21:50:45,540 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000003-wal.1731621045411.temp 2024-11-14T21:50:45,540 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:45,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741934_1113 (size=5223) 2024-11-14T21:50:45,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741934_1113 (size=5223) 2024-11-14T21:50:45,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741934_1113 (size=5223) 2024-11-14T21:50:45,549 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000003-wal.1731621045411.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:45,551 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000003-wal.1731621045411.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000035 2024-11-14T21:50:45,551 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-14T21:50:45,551 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411, journal: Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411, size=5.7 K (5869bytes) at 1731621045533Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 so closing down at 1731621045537 (+4 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000003-wal.1731621045411.temp at 1731621045540 (+3 ms)3 split writer threads finished at 1731621045540Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000003-wal.1731621045411.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731621045550 (+10 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000003-wal.1731621045411.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000035 at 1731621045551 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731621045551 2024-11-14T21:50:45,553 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045411 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621045411 2024-11-14T21:50:45,553 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000035 2024-11-14T21:50:45,554 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:45,555 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:45,568 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555, exclude list is [], retry=0 2024-11-14T21:50:45,571 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:45,571 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:45,571 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:45,573 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 2024-11-14T21:50:45,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40169:40169),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:34769:34769)] 2024-11-14T21:50:45,573 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4dd8f9bce73d9083aae557fcb2ca7d14, NAME => 'testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.', STARTKEY => '', ENDKEY => ''} 2024-11-14T21:50:45,573 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:45,574 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,574 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,575 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,576 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName a 2024-11-14T21:50:45,576 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,582 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/a/94c099fa9cc64dc09465e1fd9b5c9669 2024-11-14T21:50:45,583 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,583 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,584 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName b 2024-11-14T21:50:45,584 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,584 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,585 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,586 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName c 2024-11-14T21:50:45,586 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:45,586 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:45,586 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,589 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000035 2024-11-14T21:50:45,592 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000035 2024-11-14T21:50:45,592 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4dd8f9bce73d9083aae557fcb2ca7d14 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-14T21:50:45,607 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/b/f6fcfe6846234538a81c028befe25163 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731621045498/Put/seqid=0 2024-11-14T21:50:45,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741936_1115 (size=5958) 2024-11-14T21:50:45,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741936_1115 (size=5958) 2024-11-14T21:50:45,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/b/f6fcfe6846234538a81c028befe25163 2024-11-14T21:50:45,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741936_1115 (size=5958) 2024-11-14T21:50:45,635 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/c/c79167e209ca4e7891799c613f7124a7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731621045504/Put/seqid=0 2024-11-14T21:50:45,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741937_1116 (size=5958) 2024-11-14T21:50:45,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741937_1116 (size=5958) 2024-11-14T21:50:45,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741937_1116 (size=5958) 2024-11-14T21:50:45,644 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/c/c79167e209ca4e7891799c613f7124a7 2024-11-14T21:50:45,651 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/b/f6fcfe6846234538a81c028befe25163 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/b/f6fcfe6846234538a81c028befe25163 2024-11-14T21:50:45,657 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/b/f6fcfe6846234538a81c028befe25163, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T21:50:45,659 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/c/c79167e209ca4e7891799c613f7124a7 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/c/c79167e209ca4e7891799c613f7124a7 2024-11-14T21:50:45,664 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/c/c79167e209ca4e7891799c613f7124a7, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T21:50:45,665 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4dd8f9bce73d9083aae557fcb2ca7d14 in 73ms, sequenceid=35, compaction requested=false; wal=null 2024-11-14T21:50:45,665 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000035 2024-11-14T21:50:45,667 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,667 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,667 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:45,669 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:45,671 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-14T21:50:45,672 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4dd8f9bce73d9083aae557fcb2ca7d14; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64858133, jitterRate=-0.03353850543498993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:45,672 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Writing region info on filesystem at 1731621045574Initializing all the Stores at 1731621045575 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045575Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045575Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621045575Obtaining lock to block concurrent updates at 1731621045592 (+17 ms)Preparing flush snapshotting stores in 4dd8f9bce73d9083aae557fcb2ca7d14 at 1731621045592Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731621045592Flushing stores of testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. at 1731621045592Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/b: creating writer at 1731621045592Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/b: appending metadata at 1731621045606 (+14 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/b: closing flushed file at 1731621045606Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/c: creating writer at 1731621045620 (+14 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/c: appending metadata at 1731621045635 (+15 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/c: closing flushed file at 1731621045635Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52d4253: reopening flushed file at 1731621045650 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22a525ea: reopening flushed file at 1731621045657 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4dd8f9bce73d9083aae557fcb2ca7d14 in 73ms, sequenceid=35, compaction requested=false; wal=null at 1731621045665 (+8 ms)Cleaning up temporary data from old regions at 1731621045667 (+2 ms)Region opened successfully at 1731621045672 (+5 ms) 2024-11-14T21:50:45,743 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555, size=0 (0bytes) 2024-11-14T21:50:45,743 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 might be still open, length is 0 2024-11-14T21:50:45,743 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 2024-11-14T21:50:45,744 WARN [IPC Server handler 3 on default port 45311 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-14T21:50:45,744 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 after 1ms 2024-11-14T21:50:47,665 INFO [master/ef788a4cade8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T21:50:47,665 INFO [master/ef788a4cade8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T21:50:47,672 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:40012 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:44645:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40012 dst: /127.0.0.1:44645 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44645 remote=/127.0.0.1:40012]. Total timeout mills is 60000, 58030 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:47,672 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:49806 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:32913:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49806 dst: /127.0.0.1:32913 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:47,672 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_457615319_22 at /127.0.0.1:42724 [Receiving block BP-2030617010-172.17.0.2-1731621000517:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39681:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42724 dst: /127.0.0.1:39681 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:47,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741935_1117 (size=5215) 2024-11-14T21:50:47,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741935_1117 (size=5215) 2024-11-14T21:50:47,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741935_1117 (size=5215) 2024-11-14T21:50:49,745 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 after 4002ms 2024-11-14T21:50:49,748 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 took 4005ms 2024-11-14T21:50:49,750 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555; continuing. 2024-11-14T21:50:49,750 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 so closing down 2024-11-14T21:50:49,750 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T21:50:49,752 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731621045555.temp 2024-11-14T21:50:49,753 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000037-wal.1731621045555.temp 2024-11-14T21:50:49,753 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T21:50:49,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741938_1118 (size=5223) 2024-11-14T21:50:49,767 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000037-wal.1731621045555.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T21:50:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741938_1118 (size=5223) 2024-11-14T21:50:49,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741938_1118 (size=5223) 2024-11-14T21:50:49,768 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000037-wal.1731621045555.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000066 2024-11-14T21:50:49,769 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T21:50:49,769 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555, journal: Splitting hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555, size=0 (0bytes) at 1731621045743Finishing writing output for hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 so closing down at 1731621049750 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000037-wal.1731621045555.temp at 1731621049753 (+3 ms)3 split writer threads finished at 1731621049753Closed recovered edits writer path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000037-wal.1731621045555.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731621049767 (+14 ms)Rename recovered edits hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000037-wal.1731621045555.temp to hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000066 at 1731621049768 (+1 ms)Processed 30 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555, size=0, length=0, corrupted=false, cancelled=false at 1731621049769 (+1 ms) 2024-11-14T21:50:49,771 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 to hdfs://localhost:45311/hbase/oldWALs/wal.1731621045555 2024-11-14T21:50:49,772 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000066 2024-11-14T21:50:49,772 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T21:50:49,774 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45311/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357, archiveDir=hdfs://localhost:45311/hbase/oldWALs, maxLogs=32 2024-11-14T21:50:49,789 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621049774, exclude list is [], retry=0 2024-11-14T21:50:49,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32913,DS-13a2ab8f-ec81-4d77-adb9-1c7a85cd93d0,DISK] 2024-11-14T21:50:49,793 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39681,DS-cc2247b8-7b78-4297-b18c-eecee0d0563e,DISK] 2024-11-14T21:50:49,793 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44645,DS-a651bf80-bdd1-4033-86ca-671a3ec818a6,DISK] 2024-11-14T21:50:49,795 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621049774 2024-11-14T21:50:49,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34769:34769),(127.0.0.1/127.0.0.1:43637:43637),(127.0.0.1/127.0.0.1:40169:40169)] 2024-11-14T21:50:49,797 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T21:50:49,798 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:49,799 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName a 2024-11-14T21:50:49,799 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:49,805 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/a/94c099fa9cc64dc09465e1fd9b5c9669 2024-11-14T21:50:49,806 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:49,806 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:49,807 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName b 2024-11-14T21:50:49,807 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:49,812 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/b/f6fcfe6846234538a81c028befe25163 2024-11-14T21:50:49,813 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:49,813 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:49,814 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dd8f9bce73d9083aae557fcb2ca7d14 columnFamilyName c 2024-11-14T21:50:49,814 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T21:50:49,819 DEBUG [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/c/c79167e209ca4e7891799c613f7124a7 2024-11-14T21:50:49,819 INFO [StoreOpener-4dd8f9bce73d9083aae557fcb2ca7d14-1 {}] regionserver.HStore(327): Store=4dd8f9bce73d9083aae557fcb2ca7d14/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T21:50:49,819 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:49,820 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:49,822 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:49,822 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000066 2024-11-14T21:50:49,830 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000066 2024-11-14T21:50:49,830 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4dd8f9bce73d9083aae557fcb2ca7d14 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T21:50:49,849 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/a/baffe01817884e108c888f05b6fd62e5 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731621045680/Put/seqid=0 2024-11-14T21:50:49,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741940_1120 (size=5958) 2024-11-14T21:50:49,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741940_1120 (size=5958) 2024-11-14T21:50:49,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741940_1120 (size=5958) 2024-11-14T21:50:49,880 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/a/baffe01817884e108c888f05b6fd62e5 2024-11-14T21:50:49,910 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/b/47f6378cf8154fdeac10ee1620e02b21 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731621045688/Put/seqid=0 2024-11-14T21:50:49,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741941_1121 (size=5958) 2024-11-14T21:50:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741941_1121 (size=5958) 2024-11-14T21:50:49,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741941_1121 (size=5958) 2024-11-14T21:50:49,931 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/b/47f6378cf8154fdeac10ee1620e02b21 2024-11-14T21:50:49,961 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/c/b87907aefbf24b07b4eb52503228f517 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731621045694/Put/seqid=0 2024-11-14T21:50:49,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741942_1122 (size=5958) 2024-11-14T21:50:49,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741942_1122 (size=5958) 2024-11-14T21:50:49,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741942_1122 (size=5958) 2024-11-14T21:50:49,986 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/c/b87907aefbf24b07b4eb52503228f517 2024-11-14T21:50:50,007 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/a/baffe01817884e108c888f05b6fd62e5 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/a/baffe01817884e108c888f05b6fd62e5 2024-11-14T21:50:50,015 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/a/baffe01817884e108c888f05b6fd62e5, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T21:50:50,016 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/b/47f6378cf8154fdeac10ee1620e02b21 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/b/47f6378cf8154fdeac10ee1620e02b21 2024-11-14T21:50:50,023 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/b/47f6378cf8154fdeac10ee1620e02b21, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T21:50:50,027 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/.tmp/c/b87907aefbf24b07b4eb52503228f517 as hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/c/b87907aefbf24b07b4eb52503228f517 2024-11-14T21:50:50,059 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/c/b87907aefbf24b07b4eb52503228f517, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T21:50:50,059 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4dd8f9bce73d9083aae557fcb2ca7d14 in 229ms, sequenceid=66, compaction requested=false; wal=null 2024-11-14T21:50:50,061 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/0000000000000000066 2024-11-14T21:50:50,062 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:50,062 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:50,064 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T21:50:50,065 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4dd8f9bce73d9083aae557fcb2ca7d14 2024-11-14T21:50:50,071 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/hbase/data/default/testReplayEditsWrittenViaHRegion/4dd8f9bce73d9083aae557fcb2ca7d14/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-14T21:50:50,073 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4dd8f9bce73d9083aae557fcb2ca7d14; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59744797, jitterRate=-0.10973314940929413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T21:50:50,073 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Writing region info on filesystem at 1731621049797Initializing all the Stores at 1731621049798 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621049798Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621049798Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731621049798Obtaining lock to block concurrent updates at 1731621049830 (+32 ms)Preparing flush snapshotting stores in 4dd8f9bce73d9083aae557fcb2ca7d14 at 1731621049830Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731621049830Flushing stores of testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. at 1731621049830Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/a: creating writer at 1731621049830Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/a: appending metadata at 1731621049848 (+18 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/a: closing flushed file at 1731621049848Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/b: creating writer at 1731621049888 (+40 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/b: appending metadata at 1731621049909 (+21 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/b: closing flushed file at 1731621049909Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/c: creating writer at 1731621049938 (+29 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/c: appending metadata at 1731621049960 (+22 ms)Flushing 4dd8f9bce73d9083aae557fcb2ca7d14/c: closing flushed file at 1731621049960Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7aed29f3: reopening flushed file at 1731621050003 (+43 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76eba620: reopening flushed file at 1731621050015 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67de8fe9: reopening flushed file at 1731621050023 (+8 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4dd8f9bce73d9083aae557fcb2ca7d14 in 229ms, sequenceid=66, compaction requested=false; wal=null at 1731621050059 (+36 ms)Cleaning up temporary data from old regions at 1731621050062 (+3 ms)Region opened successfully at 1731621050073 (+11 ms) 2024-11-14T21:50:50,091 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4dd8f9bce73d9083aae557fcb2ca7d14, disabling compactions & flushes 2024-11-14T21:50:50,091 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:50,091 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:50,091 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. after waiting 0 ms 2024-11-14T21:50:50,091 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:50,107 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731621045358.4dd8f9bce73d9083aae557fcb2ca7d14. 2024-11-14T21:50:50,107 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4dd8f9bce73d9083aae557fcb2ca7d14: Waiting for close lock at 1731621050090Disabling compacts and flushes for region at 1731621050090Disabling writes for close at 1731621050091 (+1 ms)Writing region close event to WAL at 1731621050107 (+16 ms)Closed at 1731621050107 2024-11-14T21:50:50,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741939_1119 (size=93) 2024-11-14T21:50:50,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741939_1119 (size=93) 2024-11-14T21:50:50,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741939_1119 (size=93) 2024-11-14T21:50:50,114 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T21:50:50,114 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731621049774) 2024-11-14T21:50:50,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=446 (was 441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611421407_22 at /127.0.0.1:53500 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:45311 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611421407_22 at /127.0.0.1:45432 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45311 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611421407_22 at /127.0.0.1:32858 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=670 (was 676), ProcessCount=11 (was 11), AvailableMemoryMB=3699 (was 1792) - AvailableMemoryMB LEAK? - 2024-11-14T21:50:50,135 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-14T21:50:50,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T21:50:50,136 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:50:50,136 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:50:50,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T21:50:50,137 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T21:50:50,137 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1914149156, stopped=false 2024-11-14T21:50:50,137 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef788a4cade8,37243,1731621003759 2024-11-14T21:50:50,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:50,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:50,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T21:50:50,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:50,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:50,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:50,140 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T21:50:50,140 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:50,140 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:50,140 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T21:50:50,141 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T21:50:50,141 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:50:50,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,141 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef788a4cade8,39395,1731621004622' ***** 2024-11-14T21:50:50,141 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T21:50:50,141 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(3091): Received CLOSE for 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(959): stopping server ef788a4cade8,39395,1731621004622 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T21:50:50,142 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 264804e76ef9209d727464aa496291f4, disabling compactions & flushes 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef788a4cade8:39395. 2024-11-14T21:50:50,142 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:50,142 DEBUG [RS:0;ef788a4cade8:39395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:50:50,142 DEBUG [RS:0;ef788a4cade8:39395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,142 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:50,142 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. after waiting 0 ms 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T21:50:50,142 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T21:50:50,142 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T21:50:50,142 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T21:50:50,143 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T21:50:50,143 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1325): Online Regions={264804e76ef9209d727464aa496291f4=testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T21:50:50,143 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef788a4cade8,36167,1731621004776' ***** 2024-11-14T21:50:50,143 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T21:50:50,143 DEBUG [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 264804e76ef9209d727464aa496291f4 2024-11-14T21:50:50,143 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T21:50:50,143 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T21:50:50,143 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T21:50:50,143 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T21:50:50,143 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T21:50:50,143 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(959): stopping server ef788a4cade8,36167,1731621004776 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ef788a4cade8:36167. 2024-11-14T21:50:50,143 DEBUG [RS:2;ef788a4cade8:36167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T21:50:50,143 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-14T21:50:50,143 DEBUG [RS:2;ef788a4cade8:36167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,143 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(976): stopping server ef788a4cade8,36167,1731621004776; all regions closed. 2024-11-14T21:50:50,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741833_1009 (size=93) 2024-11-14T21:50:50,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741833_1009 (size=93) 2024-11-14T21:50:50,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741833_1009 (size=93) 2024-11-14T21:50:50,162 DEBUG [RS:2;ef788a4cade8:36167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs 2024-11-14T21:50:50,162 INFO [RS:2;ef788a4cade8:36167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef788a4cade8%2C36167%2C1731621004776:(num 1731621006126) 2024-11-14T21:50:50,162 DEBUG [RS:2;ef788a4cade8:36167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,162 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:50:50,163 INFO [RS:2;ef788a4cade8:36167 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T21:50:50,163 INFO [RS:2;ef788a4cade8:36167 {}] hbase.ChoreService(370): Chore service for: regionserver/ef788a4cade8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T21:50:50,163 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T21:50:50,164 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T21:50:50,164 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T21:50:50,164 INFO [RS:2;ef788a4cade8:36167 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T21:50:50,164 INFO [RS:2;ef788a4cade8:36167 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36167 2024-11-14T21:50:50,164 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/info/8d0232709ab546588abcee670ff6d14e is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4./info:regioninfo/1731621028108/Put/seqid=0 2024-11-14T21:50:50,164 INFO [regionserver/ef788a4cade8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T21:50:50,165 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/264804e76ef9209d727464aa496291f4/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-14T21:50:50,166 INFO [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:50,166 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 264804e76ef9209d727464aa496291f4: Waiting for close lock at 1731621050142Running coprocessor pre-close hooks at 1731621050142Disabling compacts and flushes for region at 1731621050142Disabling writes for close at 1731621050142Writing region close event to WAL at 1731621050144 (+2 ms)Running coprocessor post-close hooks at 1731621050166 (+22 ms)Closed at 1731621050166 2024-11-14T21:50:50,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:50:50,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef788a4cade8,36167,1731621004776 2024-11-14T21:50:50,167 INFO [RS:2;ef788a4cade8:36167 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T21:50:50,167 DEBUG [RS_CLOSE_REGION-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731621023454.264804e76ef9209d727464aa496291f4. 2024-11-14T21:50:50,169 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef788a4cade8,36167,1731621004776] 2024-11-14T21:50:50,171 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef788a4cade8,36167,1731621004776 already deleted, retry=false 2024-11-14T21:50:50,171 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef788a4cade8,36167,1731621004776 expired; onlineServers=1 2024-11-14T21:50:50,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741943_1123 (size=6778) 2024-11-14T21:50:50,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741943_1123 (size=6778) 2024-11-14T21:50:50,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741943_1123 (size=6778) 2024-11-14T21:50:50,174 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/info/8d0232709ab546588abcee670ff6d14e 2024-11-14T21:50:50,181 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/.tmp/info/8d0232709ab546588abcee670ff6d14e as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/info/8d0232709ab546588abcee670ff6d14e 2024-11-14T21:50:50,187 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/info/8d0232709ab546588abcee670ff6d14e, entries=8, sequenceid=23, filesize=6.6 K 2024-11-14T21:50:50,188 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=23, compaction requested=false 2024-11-14T21:50:50,193 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-14T21:50:50,193 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T21:50:50,193 INFO [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T21:50:50,194 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731621050143Running coprocessor pre-close hooks at 1731621050143Disabling compacts and flushes for region at 1731621050143Disabling writes for close at 1731621050143Obtaining lock to block concurrent updates at 1731621050143Preparing flush snapshotting stores in 1588230740 at 1731621050143Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731621050144 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731621050148 (+4 ms)Flushing 1588230740/info: creating writer at 1731621050148Flushing 1588230740/info: appending metadata at 1731621050163 (+15 ms)Flushing 1588230740/info: closing flushed file at 1731621050163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72619573: reopening flushed file at 1731621050180 (+17 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=23, compaction requested=false at 1731621050188 (+8 ms)Writing region close event to WAL at 1731621050189 (+1 ms)Running coprocessor post-close hooks at 1731621050193 (+4 ms)Closed at 1731621050193 2024-11-14T21:50:50,194 DEBUG [RS_CLOSE_META-regionserver/ef788a4cade8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T21:50:50,195 INFO [regionserver/ef788a4cade8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:50:50,209 INFO [regionserver/ef788a4cade8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:50:50,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:50,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36167-0x100ac6b2eed0003, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:50,270 INFO [RS:2;ef788a4cade8:36167 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T21:50:50,270 INFO [RS:2;ef788a4cade8:36167 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef788a4cade8,36167,1731621004776; zookeeper connection closed. 2024-11-14T21:50:50,270 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73fd688e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73fd688e 2024-11-14T21:50:50,343 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(976): stopping server ef788a4cade8,39395,1731621004622; all regions closed. 2024-11-14T21:50:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741893_1071 (size=2422) 2024-11-14T21:50:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741893_1071 (size=2422) 2024-11-14T21:50:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741893_1071 (size=2422) 2024-11-14T21:50:50,350 DEBUG [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs 2024-11-14T21:50:50,350 INFO [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef788a4cade8%2C39395%2C1731621004622.meta:.meta(num 1731621027165) 2024-11-14T21:50:50,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741835_1011 (size=3433) 2024-11-14T21:50:50,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741835_1011 (size=3433) 2024-11-14T21:50:50,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741835_1011 (size=3433) 2024-11-14T21:50:50,355 DEBUG [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/oldWALs 2024-11-14T21:50:50,355 INFO [RS:0;ef788a4cade8:39395 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef788a4cade8%2C39395%2C1731621004622:(num 1731621006168) 2024-11-14T21:50:50,355 DEBUG [RS:0;ef788a4cade8:39395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T21:50:50,355 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T21:50:50,355 INFO [RS:0;ef788a4cade8:39395 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T21:50:50,355 INFO [RS:0;ef788a4cade8:39395 {}] hbase.ChoreService(370): Chore service for: regionserver/ef788a4cade8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T21:50:50,355 INFO [RS:0;ef788a4cade8:39395 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T21:50:50,355 INFO [regionserver/ef788a4cade8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T21:50:50,356 INFO [RS:0;ef788a4cade8:39395 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39395 2024-11-14T21:50:50,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef788a4cade8,39395,1731621004622 2024-11-14T21:50:50,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T21:50:50,359 INFO [RS:0;ef788a4cade8:39395 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T21:50:50,361 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef788a4cade8,39395,1731621004622] 2024-11-14T21:50:50,362 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef788a4cade8,39395,1731621004622 already deleted, retry=false 2024-11-14T21:50:50,362 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef788a4cade8,39395,1731621004622 expired; onlineServers=0 2024-11-14T21:50:50,363 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef788a4cade8,37243,1731621003759' ***** 2024-11-14T21:50:50,363 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T21:50:50,363 INFO [M:0;ef788a4cade8:37243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T21:50:50,363 INFO [M:0;ef788a4cade8:37243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T21:50:50,363 DEBUG [M:0;ef788a4cade8:37243 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T21:50:50,363 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T21:50:50,363 DEBUG [M:0;ef788a4cade8:37243 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T21:50:50,363 DEBUG [master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.large.0-1731621005830 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.large.0-1731621005830,5,FailOnTimeoutGroup] 2024-11-14T21:50:50,363 DEBUG [master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.small.0-1731621005831 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef788a4cade8:0:becomeActiveMaster-HFileCleaner.small.0-1731621005831,5,FailOnTimeoutGroup] 2024-11-14T21:50:50,363 INFO [M:0;ef788a4cade8:37243 {}] hbase.ChoreService(370): Chore service for: master/ef788a4cade8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T21:50:50,364 INFO [M:0;ef788a4cade8:37243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T21:50:50,364 DEBUG [M:0;ef788a4cade8:37243 {}] master.HMaster(1795): Stopping service threads 2024-11-14T21:50:50,364 INFO [M:0;ef788a4cade8:37243 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T21:50:50,364 INFO [M:0;ef788a4cade8:37243 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T21:50:50,364 INFO [M:0;ef788a4cade8:37243 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T21:50:50,365 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T21:50:50,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T21:50:50,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T21:50:50,365 DEBUG [M:0;ef788a4cade8:37243 {}] zookeeper.ZKUtil(347): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T21:50:50,365 WARN [M:0;ef788a4cade8:37243 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T21:50:50,366 INFO [M:0;ef788a4cade8:37243 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/.lastflushedseqids 2024-11-14T21:50:50,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741944_1124 (size=119) 2024-11-14T21:50:50,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741944_1124 (size=119) 2024-11-14T21:50:50,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741944_1124 (size=119) 2024-11-14T21:50:50,379 INFO [M:0;ef788a4cade8:37243 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T21:50:50,379 INFO [M:0;ef788a4cade8:37243 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T21:50:50,379 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T21:50:50,379 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:50,379 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:50,379 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T21:50:50,379 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:50,379 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-14T21:50:50,395 DEBUG [M:0;ef788a4cade8:37243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/39c04faaffeb45c0837540a583116c20 is 82, key is hbase:meta,,1/info:regioninfo/1731621027407/Put/seqid=0 2024-11-14T21:50:50,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741945_1125 (size=6063) 2024-11-14T21:50:50,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741945_1125 (size=6063) 2024-11-14T21:50:50,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741945_1125 (size=6063) 2024-11-14T21:50:50,402 INFO [M:0;ef788a4cade8:37243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/39c04faaffeb45c0837540a583116c20 2024-11-14T21:50:50,422 DEBUG [M:0;ef788a4cade8:37243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/62983e0493eb4e6ca7af47bfc7c902db is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731621023964/Put/seqid=0 2024-11-14T21:50:50,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741946_1126 (size=7908) 2024-11-14T21:50:50,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741946_1126 (size=7908) 2024-11-14T21:50:50,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741946_1126 (size=7908) 2024-11-14T21:50:50,429 INFO [M:0;ef788a4cade8:37243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/62983e0493eb4e6ca7af47bfc7c902db 2024-11-14T21:50:50,434 INFO [M:0;ef788a4cade8:37243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62983e0493eb4e6ca7af47bfc7c902db 2024-11-14T21:50:50,449 DEBUG [M:0;ef788a4cade8:37243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1766c46d368443e6ae70c9575bfccbe5 is 69, key is ef788a4cade8,36167,1731621004776/rs:state/1731621005867/Put/seqid=0 2024-11-14T21:50:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741947_1127 (size=5445) 2024-11-14T21:50:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741947_1127 (size=5445) 2024-11-14T21:50:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741947_1127 (size=5445) 2024-11-14T21:50:50,456 INFO [M:0;ef788a4cade8:37243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1766c46d368443e6ae70c9575bfccbe5 2024-11-14T21:50:50,461 INFO [M:0;ef788a4cade8:37243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1766c46d368443e6ae70c9575bfccbe5 2024-11-14T21:50:50,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:50,461 INFO [RS:0;ef788a4cade8:39395 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T21:50:50,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39395-0x100ac6b2eed0001, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:50,461 INFO [RS:0;ef788a4cade8:39395 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef788a4cade8,39395,1731621004622; zookeeper connection closed. 2024-11-14T21:50:50,461 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/39c04faaffeb45c0837540a583116c20 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/39c04faaffeb45c0837540a583116c20 2024-11-14T21:50:50,461 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@883173 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@883173 2024-11-14T21:50:50,462 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T21:50:50,466 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/39c04faaffeb45c0837540a583116c20, entries=14, sequenceid=207, filesize=5.9 K 2024-11-14T21:50:50,467 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/62983e0493eb4e6ca7af47bfc7c902db as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/62983e0493eb4e6ca7af47bfc7c902db 2024-11-14T21:50:50,471 INFO [M:0;ef788a4cade8:37243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62983e0493eb4e6ca7af47bfc7c902db 2024-11-14T21:50:50,471 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/62983e0493eb4e6ca7af47bfc7c902db, entries=21, sequenceid=207, filesize=7.7 K 2024-11-14T21:50:50,472 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1766c46d368443e6ae70c9575bfccbe5 as hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1766c46d368443e6ae70c9575bfccbe5 2024-11-14T21:50:50,476 INFO [M:0;ef788a4cade8:37243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1766c46d368443e6ae70c9575bfccbe5 2024-11-14T21:50:50,476 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45311/user/jenkins/test-data/d8957f88-c5d8-718b-6f9a-90207488eed6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1766c46d368443e6ae70c9575bfccbe5, entries=3, sequenceid=207, filesize=5.3 K 2024-11-14T21:50:50,477 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=207, compaction requested=false 2024-11-14T21:50:50,478 INFO [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T21:50:50,479 DEBUG [M:0;ef788a4cade8:37243 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731621050379Disabling compacts and flushes for region at 1731621050379Disabling writes for close at 1731621050379Obtaining lock to block concurrent updates at 1731621050379Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731621050379Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1731621050380 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731621050380Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731621050380Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731621050395 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731621050395Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731621050406 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731621050421 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731621050422 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731621050434 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731621050448 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731621050448Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29058450: reopening flushed file at 1731621050461 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f47a91e: reopening flushed file at 1731621050466 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6760a521: reopening flushed file at 1731621050471 (+5 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=207, compaction requested=false at 1731621050477 (+6 ms)Writing region close event to WAL at 1731621050478 (+1 ms)Closed at 1731621050478 2024-11-14T21:50:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39681 is added to blk_1073741830_1006 (size=99448) 2024-11-14T21:50:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32913 is added to blk_1073741830_1006 (size=99448) 2024-11-14T21:50:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44645 is added to blk_1073741830_1006 (size=99448) 2024-11-14T21:50:50,481 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T21:50:50,482 INFO [M:0;ef788a4cade8:37243 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T21:50:50,482 INFO [M:0;ef788a4cade8:37243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37243 2024-11-14T21:50:50,482 INFO [M:0;ef788a4cade8:37243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T21:50:50,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:50,586 INFO [M:0;ef788a4cade8:37243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T21:50:50,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37243-0x100ac6b2eed0000, quorum=127.0.0.1:60020, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T21:50:50,592 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621039333 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621039333 (inode 16688) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731621029870/wal.1731621039333 (inode 16688) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T21:50:50,595 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621029586 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621029586 (inode 16665) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731621028447/wal.1731621029586 (inode 16665) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T21:50:50,595 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731621018563/wal.1731621018751 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:50,597 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621040196 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621040196 (inode 16714) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731621039599/wal.1731621040196 (inode 16714) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T21:50:50,598 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731621040356/wal.1731621040499 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:50,598 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731621045357/wal.1731621045555 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:50,598 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731621009348/wal.1731621009435 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T21:50:50,600 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal.1731621008749 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal.1731621008749 (inode 16485) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731621008381/wal.1731621008749 (inode 16485) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T21:50:50,602 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal.1731621007999 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal.1731621007999 (inode 16462) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731621007236/wal.1731621007999 (inode 16462) Holder DFSClient_NONMAPREDUCE_457615319_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T21:50:50,606 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:50:50,609 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T21:50:50,609 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T21:50:50,609 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T21:50:50,609 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,STOPPED} 2024-11-14T21:50:50,612 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T21:50:50,612 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T21:50:50,615 WARN [BP-2030617010-172.17.0.2-1731621000517 heartbeating to localhost/127.0.0.1:45311 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T21:50:50,615 WARN [BP-2030617010-172.17.0.2-1731621000517 heartbeating to localhost/127.0.0.1:45311 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2030617010-172.17.0.2-1731621000517 (Datanode Uuid 80cbac55-7fc0-4e95-99ef-4db521160716) service to localhost/127.0.0.1:45311 2024-11-14T21:50:50,616 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data5/current/BP-2030617010-172.17.0.2-1731621000517 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:50:50,616 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data6/current/BP-2030617010-172.17.0.2-1731621000517 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:50:50,617 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T21:50:50,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:50:50,653 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T21:50:50,653 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T21:50:50,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T21:50:50,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,STOPPED} 2024-11-14T21:50:50,655 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T21:50:50,655 WARN [BP-2030617010-172.17.0.2-1731621000517 heartbeating to localhost/127.0.0.1:45311 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T21:50:50,655 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T21:50:50,655 WARN [BP-2030617010-172.17.0.2-1731621000517 heartbeating to localhost/127.0.0.1:45311 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2030617010-172.17.0.2-1731621000517 (Datanode Uuid 1a2405d8-c26c-47fc-9586-bd1c92159f32) service to localhost/127.0.0.1:45311 2024-11-14T21:50:50,655 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data3/current/BP-2030617010-172.17.0.2-1731621000517 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:50:50,656 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data4/current/BP-2030617010-172.17.0.2-1731621000517 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:50:50,656 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T21:50:50,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T21:50:50,658 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T21:50:50,658 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T21:50:50,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T21:50:50,659 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,STOPPED} 2024-11-14T21:50:50,660 WARN [BP-2030617010-172.17.0.2-1731621000517 heartbeating to localhost/127.0.0.1:45311 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T21:50:50,660 WARN [BP-2030617010-172.17.0.2-1731621000517 heartbeating to localhost/127.0.0.1:45311 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2030617010-172.17.0.2-1731621000517 (Datanode Uuid 7fe6d6ea-3168-4cc0-aaf1-cdfb65011532) service to localhost/127.0.0.1:45311 2024-11-14T21:50:50,661 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data1/current/BP-2030617010-172.17.0.2-1731621000517 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:50:50,661 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T21:50:50,661 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/cluster_59f76d56-3307-2669-545b-68cc1dee1c06/data/data2/current/BP-2030617010-172.17.0.2-1731621000517 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T21:50:50,661 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T21:50:50,661 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T21:50:50,667 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T21:50:50,668 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T21:50:50,668 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T21:50:50,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T21:50:50,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/627f8cdf-3181-501f-71a9-5f83c222aec3/hadoop.log.dir/,STOPPED} 2024-11-14T21:50:50,679 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T21:50:50,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down