2024-11-23 18:47:22,940 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-23 18:47:22,958 main DEBUG Took 0.014367 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 18:47:22,958 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 18:47:22,959 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 18:47:22,960 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 18:47:22,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,970 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 18:47:22,985 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,988 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,990 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,992 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,993 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,995 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,996 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,997 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,998 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:22,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:22,999 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:23,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:23,000 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:23,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:23,001 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:23,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:47:23,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:23,003 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 18:47:23,005 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:47:23,007 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 18:47:23,009 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 18:47:23,010 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 18:47:23,011 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 18:47:23,012 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 18:47:23,023 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 18:47:23,027 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 18:47:23,029 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 18:47:23,030 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 18:47:23,030 main DEBUG createAppenders(={Console}) 2024-11-23 18:47:23,031 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-23 18:47:23,031 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-23 18:47:23,032 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-23 18:47:23,033 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 18:47:23,033 main DEBUG OutputStream closed 2024-11-23 18:47:23,033 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 18:47:23,034 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 18:47:23,034 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-23 18:47:23,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 18:47:23,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 18:47:23,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 18:47:23,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 18:47:23,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 18:47:23,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 18:47:23,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 18:47:23,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 18:47:23,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 18:47:23,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 18:47:23,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 18:47:23,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 18:47:23,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 18:47:23,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 18:47:23,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 18:47:23,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 18:47:23,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 18:47:23,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 18:47:23,121 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 18:47:23,121 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-23 18:47:23,121 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 18:47:23,122 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-23T18:47:23,329 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8 2024-11-23 18:47:23,332 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 18:47:23,332 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T18:47:23,341 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-23T18:47:23,368 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T18:47:23,408 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T18:47:23,408 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T18:47:23,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T18:47:23,433 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e, deleteOnExit=true 2024-11-23T18:47:23,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T18:47:23,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/test.cache.data in system properties and HBase conf 2024-11-23T18:47:23,435 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T18:47:23,435 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir in system properties and HBase conf 2024-11-23T18:47:23,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T18:47:23,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T18:47:23,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T18:47:23,511 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T18:47:23,600 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T18:47:23,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:47:23,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T18:47:23,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T18:47:23,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:47:23,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T18:47:23,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T18:47:23,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T18:47:23,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:47:23,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T18:47:23,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/nfs.dump.dir in system properties and HBase conf 2024-11-23T18:47:23,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/java.io.tmpdir in system properties and HBase conf 2024-11-23T18:47:23,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T18:47:23,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T18:47:23,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T18:47:24,772 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T18:47:24,839 INFO [Time-limited test {}] log.Log(170): Logging initialized @2586ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T18:47:24,905 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:47:24,960 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:47:24,980 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:47:24,980 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:47:24,981 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:47:24,999 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:47:25,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b5fa2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:47:25,003 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d655053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:47:25,178 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fe9bfd3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/java.io.tmpdir/jetty-localhost-46149-hadoop-hdfs-3_4_1-tests_jar-_-any-1638715202796298691/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:47:25,183 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16695a38{HTTP/1.1, (http/1.1)}{localhost:46149} 2024-11-23T18:47:25,183 INFO [Time-limited test {}] server.Server(415): Started @2931ms 2024-11-23T18:47:25,733 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:47:25,741 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:47:25,742 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:47:25,742 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:47:25,742 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T18:47:25,743 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@302ee775{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:47:25,744 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c68adb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:47:25,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@516a39dc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/java.io.tmpdir/jetty-localhost-39665-hadoop-hdfs-3_4_1-tests_jar-_-any-14152794132422207592/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:47:25,847 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@428881d6{HTTP/1.1, (http/1.1)}{localhost:39665} 2024-11-23T18:47:25,847 INFO [Time-limited test {}] server.Server(415): Started @3595ms 2024-11-23T18:47:25,894 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:47:26,000 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:47:26,006 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:47:26,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:47:26,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:47:26,009 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:47:26,010 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2599967d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:47:26,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4925ca3e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:47:26,115 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cc5338b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/java.io.tmpdir/jetty-localhost-35641-hadoop-hdfs-3_4_1-tests_jar-_-any-14107035801587854362/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:47:26,116 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1364e1fb{HTTP/1.1, (http/1.1)}{localhost:35641} 2024-11-23T18:47:26,116 INFO [Time-limited test {}] server.Server(415): Started @3864ms 2024-11-23T18:47:26,119 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:47:26,155 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T18:47:26,159 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T18:47:26,161 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T18:47:26,161 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T18:47:26,161 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T18:47:26,162 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@626fc3bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,AVAILABLE} 2024-11-23T18:47:26,163 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@147d1a17{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T18:47:26,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ed6f3ed{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/java.io.tmpdir/jetty-localhost-36503-hadoop-hdfs-3_4_1-tests_jar-_-any-7422264046722694325/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:47:26,272 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f154eb3{HTTP/1.1, (http/1.1)}{localhost:36503} 2024-11-23T18:47:26,273 INFO [Time-limited test {}] server.Server(415): Started @4020ms 2024-11-23T18:47:26,275 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T18:47:27,566 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data3/current/BP-657037303-172.17.0.2-1732387644132/current, will proceed with Du for space computation calculation, 2024-11-23T18:47:27,566 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data1/current/BP-657037303-172.17.0.2-1732387644132/current, will proceed with Du for space computation calculation, 2024-11-23T18:47:27,566 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data2/current/BP-657037303-172.17.0.2-1732387644132/current, will proceed with Du for space computation calculation, 2024-11-23T18:47:27,566 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data4/current/BP-657037303-172.17.0.2-1732387644132/current, will proceed with Du for space computation calculation, 2024-11-23T18:47:27,603 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:47:27,603 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:47:27,640 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data6/current/BP-657037303-172.17.0.2-1732387644132/current, will proceed with Du for space computation calculation, 2024-11-23T18:47:27,640 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data5/current/BP-657037303-172.17.0.2-1732387644132/current, will proceed with Du for space computation calculation, 2024-11-23T18:47:27,650 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc242c4310dbb56e8 with lease ID 0xe0e01bc589577f6e: Processing first storage report for DS-495cf48c-cc16-484c-9f2c-99cf787132a4 from datanode DatanodeRegistration(127.0.0.1:38589, datanodeUuid=82f019bb-94fc-4511-8a11-4ed7272f1081, infoPort=36753, infoSecurePort=0, ipcPort=36491, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132) 2024-11-23T18:47:27,651 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc242c4310dbb56e8 with lease ID 0xe0e01bc589577f6e: from storage DS-495cf48c-cc16-484c-9f2c-99cf787132a4 node DatanodeRegistration(127.0.0.1:38589, datanodeUuid=82f019bb-94fc-4511-8a11-4ed7272f1081, infoPort=36753, infoSecurePort=0, ipcPort=36491, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-23T18:47:27,652 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x238b6c58d856698a with lease ID 0xe0e01bc589577f6f: Processing first storage report for DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c from datanode DatanodeRegistration(127.0.0.1:34245, datanodeUuid=c4ccde46-6de8-4e29-807c-8c9d02ca3b86, infoPort=45113, infoSecurePort=0, ipcPort=45623, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132) 2024-11-23T18:47:27,652 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x238b6c58d856698a with lease ID 0xe0e01bc589577f6f: from storage DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c node DatanodeRegistration(127.0.0.1:34245, datanodeUuid=c4ccde46-6de8-4e29-807c-8c9d02ca3b86, infoPort=45113, infoSecurePort=0, ipcPort=45623, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:47:27,652 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc242c4310dbb56e8 with lease ID 0xe0e01bc589577f6e: Processing first storage report for DS-decc1801-d31e-4482-b051-32a0f3795537 from datanode DatanodeRegistration(127.0.0.1:38589, datanodeUuid=82f019bb-94fc-4511-8a11-4ed7272f1081, infoPort=36753, infoSecurePort=0, ipcPort=36491, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132) 2024-11-23T18:47:27,652 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc242c4310dbb56e8 with lease ID 0xe0e01bc589577f6e: from storage DS-decc1801-d31e-4482-b051-32a0f3795537 node DatanodeRegistration(127.0.0.1:38589, datanodeUuid=82f019bb-94fc-4511-8a11-4ed7272f1081, infoPort=36753, infoSecurePort=0, ipcPort=36491, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:47:27,653 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x238b6c58d856698a with lease ID 0xe0e01bc589577f6f: Processing first storage report for DS-a06ac566-b69c-4f9c-be5a-0c3f7c591416 from datanode DatanodeRegistration(127.0.0.1:34245, datanodeUuid=c4ccde46-6de8-4e29-807c-8c9d02ca3b86, infoPort=45113, infoSecurePort=0, ipcPort=45623, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132) 2024-11-23T18:47:27,653 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x238b6c58d856698a with lease ID 0xe0e01bc589577f6f: from storage DS-a06ac566-b69c-4f9c-be5a-0c3f7c591416 node DatanodeRegistration(127.0.0.1:34245, datanodeUuid=c4ccde46-6de8-4e29-807c-8c9d02ca3b86, infoPort=45113, infoSecurePort=0, ipcPort=45623, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:47:27,658 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T18:47:27,664 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41274d665f1ace1b with lease ID 0xe0e01bc589577f70: Processing first storage report for DS-93877182-6065-4aee-9cc8-d3467a2d0e87 from datanode DatanodeRegistration(127.0.0.1:39151, datanodeUuid=062048f1-dc65-4c1b-8b20-b35c20c68693, infoPort=37381, infoSecurePort=0, ipcPort=41295, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132) 2024-11-23T18:47:27,664 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41274d665f1ace1b with lease ID 0xe0e01bc589577f70: from storage DS-93877182-6065-4aee-9cc8-d3467a2d0e87 node DatanodeRegistration(127.0.0.1:39151, datanodeUuid=062048f1-dc65-4c1b-8b20-b35c20c68693, infoPort=37381, infoSecurePort=0, ipcPort=41295, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T18:47:27,664 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41274d665f1ace1b with lease ID 0xe0e01bc589577f70: Processing first storage report for DS-5479aa93-508d-4b05-a504-357599602fc3 from datanode DatanodeRegistration(127.0.0.1:39151, datanodeUuid=062048f1-dc65-4c1b-8b20-b35c20c68693, infoPort=37381, infoSecurePort=0, ipcPort=41295, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132) 2024-11-23T18:47:27,664 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41274d665f1ace1b with lease ID 0xe0e01bc589577f70: from storage DS-5479aa93-508d-4b05-a504-357599602fc3 node DatanodeRegistration(127.0.0.1:39151, datanodeUuid=062048f1-dc65-4c1b-8b20-b35c20c68693, infoPort=37381, infoSecurePort=0, ipcPort=41295, storageInfo=lv=-57;cid=testClusterID;nsid=837285042;c=1732387644132), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T18:47:27,706 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8 2024-11-23T18:47:27,767 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/zookeeper_0, clientPort=63996, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T18:47:27,778 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63996 2024-11-23T18:47:27,788 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:27,791 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:28,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:47:28,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:47:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741825_1001 (size=7) 2024-11-23T18:47:28,419 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f with version=8 2024-11-23T18:47:28,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/hbase-staging 2024-11-23T18:47:28,709 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:47:28,718 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:28,718 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:28,722 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:47:28,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:28,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:47:28,843 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T18:47:28,897 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T18:47:28,907 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T18:47:28,910 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:47:28,934 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 57364 (auto-detected) 2024-11-23T18:47:28,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T18:47:28,953 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37311 2024-11-23T18:47:28,972 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37311 connecting to ZooKeeper ensemble=127.0.0.1:63996 2024-11-23T18:47:29,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373110x0, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:47:29,115 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37311-0x1016905acb30000 connected 2024-11-23T18:47:29,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,228 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:47:29,240 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f, hbase.cluster.distributed=false 2024-11-23T18:47:29,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:47:29,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37311 2024-11-23T18:47:29,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37311 2024-11-23T18:47:29,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37311 2024-11-23T18:47:29,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37311 2024-11-23T18:47:29,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37311 2024-11-23T18:47:29,379 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:47:29,381 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,381 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,381 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:47:29,381 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,381 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:47:29,384 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:47:29,387 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:47:29,388 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36631 2024-11-23T18:47:29,390 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36631 connecting to ZooKeeper ensemble=127.0.0.1:63996 2024-11-23T18:47:29,391 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,395 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366310x0, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:47:29,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366310x0, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:47:29,411 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36631-0x1016905acb30001 connected 2024-11-23T18:47:29,415 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:47:29,424 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:47:29,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:47:29,434 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:47:29,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36631 2024-11-23T18:47:29,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36631 2024-11-23T18:47:29,438 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36631 2024-11-23T18:47:29,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36631 2024-11-23T18:47:29,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36631 2024-11-23T18:47:29,456 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:47:29,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,457 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:47:29,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:47:29,458 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:47:29,458 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:47:29,459 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37545 2024-11-23T18:47:29,461 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37545 connecting to ZooKeeper ensemble=127.0.0.1:63996 2024-11-23T18:47:29,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,465 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375450x0, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:47:29,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:375450x0, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:47:29,480 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37545-0x1016905acb30002 connected 2024-11-23T18:47:29,481 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:47:29,481 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:47:29,482 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:47:29,484 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:47:29,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37545 2024-11-23T18:47:29,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37545 2024-11-23T18:47:29,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37545 2024-11-23T18:47:29,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37545 2024-11-23T18:47:29,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37545 2024-11-23T18:47:29,503 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2eda60943ea1:0 server-side Connection retries=45 2024-11-23T18:47:29,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,504 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T18:47:29,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T18:47:29,504 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T18:47:29,504 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T18:47:29,505 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:47:29,505 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41643 2024-11-23T18:47:29,507 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41643 connecting to ZooKeeper ensemble=127.0.0.1:63996 2024-11-23T18:47:29,508 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,511 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416430x0, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T18:47:29,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:416430x0, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:47:29,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41643-0x1016905acb30003 connected 2024-11-23T18:47:29,527 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T18:47:29,529 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T18:47:29,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T18:47:29,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T18:47:29,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41643 2024-11-23T18:47:29,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41643 2024-11-23T18:47:29,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41643 2024-11-23T18:47:29,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41643 2024-11-23T18:47:29,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41643 2024-11-23T18:47:29,553 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2eda60943ea1:37311 2024-11-23T18:47:29,555 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2eda60943ea1,37311,1732387648561 2024-11-23T18:47:29,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,570 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2eda60943ea1,37311,1732387648561 2024-11-23T18:47:29,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:47:29,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:47:29,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T18:47:29,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,601 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T18:47:29,602 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2eda60943ea1,37311,1732387648561 from backup master directory 2024-11-23T18:47:29,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2eda60943ea1,37311,1732387648561 2024-11-23T18:47:29,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T18:47:29,611 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:47:29,611 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2eda60943ea1,37311,1732387648561 2024-11-23T18:47:29,613 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T18:47:29,615 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T18:47:29,671 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/hbase.id] with ID: 36796ac2-6f7b-4ddf-9283-66f1c579705e 2024-11-23T18:47:29,671 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/.tmp/hbase.id 2024-11-23T18:47:29,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:47:29,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:47:29,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741826_1002 (size=42) 2024-11-23T18:47:29,688 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/.tmp/hbase.id]:[hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/hbase.id] 2024-11-23T18:47:29,735 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T18:47:29,739 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T18:47:29,758 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-23T18:47:29,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:29,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:47:29,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:47:29,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741827_1003 (size=196) 2024-11-23T18:47:29,806 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:47:29,807 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T18:47:29,812 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:29,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:47:29,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:47:29,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741828_1004 (size=1189) 2024-11-23T18:47:29,860 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store 2024-11-23T18:47:29,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:47:29,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:47:29,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741829_1005 (size=34) 2024-11-23T18:47:29,890 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T18:47:29,893 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:29,894 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:47:29,895 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:47:29,895 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:47:29,896 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:47:29,896 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:47:29,896 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:47:29,897 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387649894Disabling compacts and flushes for region at 1732387649894Disabling writes for close at 1732387649896 (+2 ms)Writing region close event to WAL at 1732387649896Closed at 1732387649896 2024-11-23T18:47:29,900 WARN [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/.initializing 2024-11-23T18:47:29,900 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/WALs/2eda60943ea1,37311,1732387648561 2024-11-23T18:47:29,908 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:29,922 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C37311%2C1732387648561, suffix=, logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/WALs/2eda60943ea1,37311,1732387648561, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/oldWALs, maxLogs=10 2024-11-23T18:47:29,949 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/WALs/2eda60943ea1,37311,1732387648561/2eda60943ea1%2C37311%2C1732387648561.1732387649927, exclude list is [], retry=0 2024-11-23T18:47:29,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:29,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:29,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:29,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:29,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T18:47:30,009 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/WALs/2eda60943ea1,37311,1732387648561/2eda60943ea1%2C37311%2C1732387648561.1732387649927 2024-11-23T18:47:30,010 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:30,010 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:30,011 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:30,014 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,014 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T18:47:30,072 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:30,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T18:47:30,078 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,079 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:30,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T18:47:30,082 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:30,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T18:47:30,086 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:30,087 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,091 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,092 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,097 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,098 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,101 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:47:30,104 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T18:47:30,109 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:30,110 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71384508, jitterRate=0.06371206045150757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:47:30,118 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732387650025Initializing all the Stores at 1732387650028 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387650028Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387650029 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387650029Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387650029Cleaning up temporary data from old regions at 1732387650098 (+69 ms)Region opened successfully at 1732387650117 (+19 ms) 2024-11-23T18:47:30,119 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T18:47:30,149 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331b6e9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:47:30,176 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T18:47:30,185 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T18:47:30,185 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T18:47:30,188 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T18:47:30,189 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T18:47:30,194 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T18:47:30,194 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T18:47:30,223 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T18:47:30,232 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T18:47:30,284 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T18:47:30,289 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T18:47:30,292 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T18:47:30,304 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T18:47:30,307 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T18:47:30,311 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T18:47:30,321 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T18:47:30,323 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T18:47:30,336 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T18:47:30,356 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T18:47:30,367 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T18:47:30,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:47:30,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:47:30,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:47:30,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T18:47:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,382 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2eda60943ea1,37311,1732387648561, sessionid=0x1016905acb30000, setting cluster-up flag (Was=false) 2024-11-23T18:47:30,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,453 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T18:47:30,459 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,37311,1732387648561 2024-11-23T18:47:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:30,515 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T18:47:30,519 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2eda60943ea1,37311,1732387648561 2024-11-23T18:47:30,528 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T18:47:30,540 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(746): ClusterId : 36796ac2-6f7b-4ddf-9283-66f1c579705e 2024-11-23T18:47:30,540 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(746): ClusterId : 36796ac2-6f7b-4ddf-9283-66f1c579705e 2024-11-23T18:47:30,540 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(746): ClusterId : 36796ac2-6f7b-4ddf-9283-66f1c579705e 2024-11-23T18:47:30,542 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:47:30,542 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:47:30,542 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T18:47:30,565 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:47:30,565 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:47:30,565 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T18:47:30,565 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:47:30,565 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:47:30,565 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T18:47:30,575 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:47:30,575 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:47:30,575 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T18:47:30,576 DEBUG [RS:0;2eda60943ea1:36631 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fab0e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:47:30,576 DEBUG [RS:2;2eda60943ea1:41643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79af0278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:47:30,576 DEBUG [RS:1;2eda60943ea1:37545 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39cdcaaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2eda60943ea1/172.17.0.2:0 2024-11-23T18:47:30,588 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2eda60943ea1:37545 2024-11-23T18:47:30,591 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2eda60943ea1:41643 2024-11-23T18:47:30,591 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:47:30,591 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:47:30,592 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:47:30,592 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:47:30,592 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:47:30,592 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:47:30,593 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2eda60943ea1:36631 2024-11-23T18:47:30,593 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T18:47:30,593 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T18:47:30,594 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T18:47:30,594 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37311,1732387648561 with port=41643, startcode=1732387649503 2024-11-23T18:47:30,594 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37311,1732387648561 with port=37545, startcode=1732387649456 2024-11-23T18:47:30,595 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37311,1732387648561 with port=36631, startcode=1732387649346 2024-11-23T18:47:30,604 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T18:47:30,605 DEBUG [RS:2;2eda60943ea1:41643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:47:30,605 DEBUG [RS:0;2eda60943ea1:36631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:47:30,605 DEBUG [RS:1;2eda60943ea1:37545 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T18:47:30,613 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T18:47:30,620 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T18:47:30,626 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2eda60943ea1,37311,1732387648561 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T18:47:30,633 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:47:30,633 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:47:30,633 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:47:30,633 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2eda60943ea1:0, corePoolSize=5, maxPoolSize=5 2024-11-23T18:47:30,634 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2eda60943ea1:0, corePoolSize=10, maxPoolSize=10 2024-11-23T18:47:30,634 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,634 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:47:30,634 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,636 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732387680636 2024-11-23T18:47:30,638 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:47:30,638 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47023, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:47:30,638 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60945, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T18:47:30,638 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T18:47:30,640 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T18:47:30,641 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:47:30,641 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T18:47:30,644 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T18:47:30,644 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T18:47:30,645 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T18:47:30,645 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T18:47:30,645 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T18:47:30,646 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,647 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T18:47:30,646 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,649 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T18:47:30,651 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T18:47:30,651 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T18:47:30,653 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T18:47:30,654 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T18:47:30,654 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T18:47:30,655 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T18:47:30,660 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387650656,5,FailOnTimeoutGroup] 2024-11-23T18:47:30,661 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387650660,5,FailOnTimeoutGroup] 2024-11-23T18:47:30,661 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,662 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T18:47:30,677 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,678 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:47:30,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:47:30,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741831_1007 (size=1321) 2024-11-23T18:47:30,682 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T18:47:30,683 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f 2024-11-23T18:47:30,692 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T18:47:30,692 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T18:47:30,692 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T18:47:30,692 WARN [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T18:47:30,692 WARN [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T18:47:30,692 WARN [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T18:47:30,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:47:30,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:47:30,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741832_1008 (size=32) 2024-11-23T18:47:30,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:30,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:47:30,704 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:47:30,704 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:30,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:47:30,707 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:47:30,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:30,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:47:30,711 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:47:30,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:30,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:47:30,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:47:30,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:30,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:30,716 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:47:30,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740 2024-11-23T18:47:30,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740 2024-11-23T18:47:30,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:47:30,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:47:30,724 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:47:30,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:47:30,732 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:30,733 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63072738, jitterRate=-0.060142964124679565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:47:30,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732387650699Initializing all the Stores at 1732387650700 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387650700Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387650700Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387650701 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387650701Cleaning up temporary data from old regions at 1732387650723 (+22 ms)Region opened successfully at 1732387650738 (+15 ms) 2024-11-23T18:47:30,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:47:30,739 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:47:30,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:47:30,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:47:30,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:47:30,741 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:47:30,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387650739Disabling compacts and flushes for region at 1732387650739Disabling writes for close at 1732387650739Writing region close event to WAL at 1732387650740 (+1 ms)Closed at 1732387650741 (+1 ms) 2024-11-23T18:47:30,745 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:47:30,745 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T18:47:30,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:47:30,763 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:47:30,766 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T18:47:30,794 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37311,1732387648561 with port=36631, startcode=1732387649346 2024-11-23T18:47:30,794 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37311,1732387648561 with port=37545, startcode=1732387649456 2024-11-23T18:47:30,794 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(2659): reportForDuty to master=2eda60943ea1,37311,1732387648561 with port=41643, startcode=1732387649503 2024-11-23T18:47:30,795 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,36631,1732387649346 2024-11-23T18:47:30,798 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:30,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,41643,1732387649503 2024-11-23T18:47:30,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:30,805 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f 2024-11-23T18:47:30,805 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45809 2024-11-23T18:47:30,806 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:47:30,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2eda60943ea1,37545,1732387649456 2024-11-23T18:47:30,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] master.ServerManager(517): Registering regionserver=2eda60943ea1,37545,1732387649456 2024-11-23T18:47:30,808 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f 2024-11-23T18:47:30,808 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45809 2024-11-23T18:47:30,808 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:47:30,811 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f 2024-11-23T18:47:30,811 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45809 2024-11-23T18:47:30,811 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T18:47:30,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:47:30,861 DEBUG [RS:0;2eda60943ea1:36631 {}] zookeeper.ZKUtil(111): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,36631,1732387649346 2024-11-23T18:47:30,862 WARN [RS:0;2eda60943ea1:36631 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:47:30,862 INFO [RS:0;2eda60943ea1:36631 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:30,862 DEBUG [RS:2;2eda60943ea1:41643 {}] zookeeper.ZKUtil(111): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,41643,1732387649503 2024-11-23T18:47:30,862 DEBUG [RS:1;2eda60943ea1:37545 {}] zookeeper.ZKUtil(111): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2eda60943ea1,37545,1732387649456 2024-11-23T18:47:30,862 WARN [RS:2;2eda60943ea1:41643 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:47:30,862 WARN [RS:1;2eda60943ea1:37545 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T18:47:30,862 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346 2024-11-23T18:47:30,862 INFO [RS:1;2eda60943ea1:37545 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:30,862 INFO [RS:2;2eda60943ea1:41643 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:30,862 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503 2024-11-23T18:47:30,862 DEBUG [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,37545,1732387649456 2024-11-23T18:47:30,863 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,36631,1732387649346] 2024-11-23T18:47:30,863 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,37545,1732387649456] 2024-11-23T18:47:30,863 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2eda60943ea1,41643,1732387649503] 2024-11-23T18:47:30,889 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:47:30,889 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:47:30,889 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T18:47:30,904 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:47:30,904 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:47:30,905 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T18:47:30,909 INFO [RS:1;2eda60943ea1:37545 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:47:30,910 INFO [RS:0;2eda60943ea1:36631 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:47:30,910 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,910 INFO [RS:2;2eda60943ea1:41643 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:47:30,910 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,910 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,911 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:47:30,911 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:47:30,911 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T18:47:30,916 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:47:30,916 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:47:30,916 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T18:47:30,917 WARN [2eda60943ea1:37311 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T18:47:30,918 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,918 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,918 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,918 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,918 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,918 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,918 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,918 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,918 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:47:30,919 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:47:30,919 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0, corePoolSize=2, maxPoolSize=2 2024-11-23T18:47:30,919 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,919 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2eda60943ea1:0, corePoolSize=1, maxPoolSize=1 2024-11-23T18:47:30,920 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:47:30,920 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:47:30,920 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:47:30,920 DEBUG [RS:1;2eda60943ea1:37545 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:47:30,920 DEBUG [RS:2;2eda60943ea1:41643 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:47:30,920 DEBUG [RS:0;2eda60943ea1:36631 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2eda60943ea1:0, corePoolSize=3, maxPoolSize=3 2024-11-23T18:47:30,926 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37545,1732387649456-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,926 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,927 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41643,1732387649503-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:47:30,927 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,927 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36631,1732387649346-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:47:30,946 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:47:30,946 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:47:30,946 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T18:47:30,947 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,41643,1732387649503-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,948 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,36631,1732387649346-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,948 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37545,1732387649456-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,948 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,948 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,948 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,948 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.Replication(171): 2eda60943ea1,37545,1732387649456 started 2024-11-23T18:47:30,948 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.Replication(171): 2eda60943ea1,41643,1732387649503 started 2024-11-23T18:47:30,948 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.Replication(171): 2eda60943ea1,36631,1732387649346 started 2024-11-23T18:47:30,966 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,966 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,966 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:30,966 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,36631,1732387649346, RpcServer on 2eda60943ea1/172.17.0.2:36631, sessionid=0x1016905acb30001 2024-11-23T18:47:30,966 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,41643,1732387649503, RpcServer on 2eda60943ea1/172.17.0.2:41643, sessionid=0x1016905acb30003 2024-11-23T18:47:30,966 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(1482): Serving as 2eda60943ea1,37545,1732387649456, RpcServer on 2eda60943ea1/172.17.0.2:37545, sessionid=0x1016905acb30002 2024-11-23T18:47:30,967 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:47:30,967 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:47:30,967 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T18:47:30,967 DEBUG [RS:1;2eda60943ea1:37545 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,37545,1732387649456 2024-11-23T18:47:30,967 DEBUG [RS:0;2eda60943ea1:36631 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,36631,1732387649346 2024-11-23T18:47:30,967 DEBUG [RS:2;2eda60943ea1:41643 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2eda60943ea1,41643,1732387649503 2024-11-23T18:47:30,967 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,37545,1732387649456' 2024-11-23T18:47:30,967 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,36631,1732387649346' 2024-11-23T18:47:30,967 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,41643,1732387649503' 2024-11-23T18:47:30,967 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:47:30,967 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:47:30,967 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T18:47:30,968 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:47:30,968 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:47:30,968 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T18:47:30,969 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:47:30,969 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:47:30,969 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:47:30,969 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:47:30,969 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T18:47:30,969 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T18:47:30,969 DEBUG [RS:0;2eda60943ea1:36631 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,36631,1732387649346 2024-11-23T18:47:30,969 DEBUG [RS:1;2eda60943ea1:37545 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,37545,1732387649456 2024-11-23T18:47:30,969 DEBUG [RS:2;2eda60943ea1:41643 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2eda60943ea1,41643,1732387649503 2024-11-23T18:47:30,969 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,41643,1732387649503' 2024-11-23T18:47:30,969 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,37545,1732387649456' 2024-11-23T18:47:30,969 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2eda60943ea1,36631,1732387649346' 2024-11-23T18:47:30,969 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:47:30,969 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:47:30,969 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T18:47:30,970 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:47:30,970 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:47:30,970 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T18:47:30,970 DEBUG [RS:2;2eda60943ea1:41643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:47:30,970 DEBUG [RS:0;2eda60943ea1:36631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:47:30,970 DEBUG [RS:1;2eda60943ea1:37545 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T18:47:30,970 INFO [RS:2;2eda60943ea1:41643 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:47:30,970 INFO [RS:0;2eda60943ea1:36631 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:47:30,970 INFO [RS:1;2eda60943ea1:37545 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T18:47:30,970 INFO [RS:2;2eda60943ea1:41643 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:47:30,970 INFO [RS:0;2eda60943ea1:36631 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:47:30,970 INFO [RS:1;2eda60943ea1:37545 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T18:47:31,077 INFO [RS:2;2eda60943ea1:41643 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:31,077 INFO [RS:1;2eda60943ea1:37545 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:31,077 INFO [RS:0;2eda60943ea1:36631 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:31,081 INFO [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C36631%2C1732387649346, suffix=, logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs, maxLogs=32 2024-11-23T18:47:31,081 INFO [RS:1;2eda60943ea1:37545 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C37545%2C1732387649456, suffix=, logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,37545,1732387649456, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs, maxLogs=32 2024-11-23T18:47:31,081 INFO [RS:2;2eda60943ea1:41643 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C41643%2C1732387649503, suffix=, logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs, maxLogs=32 2024-11-23T18:47:31,098 DEBUG [RS:0;2eda60943ea1:36631 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346/2eda60943ea1%2C36631%2C1732387649346.1732387651084, exclude list is [], retry=0 2024-11-23T18:47:31,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:31,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:31,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:31,104 DEBUG [RS:1;2eda60943ea1:37545 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,37545,1732387649456/2eda60943ea1%2C37545%2C1732387649456.1732387651085, exclude list is [], retry=0 2024-11-23T18:47:31,104 DEBUG [RS:2;2eda60943ea1:41643 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503/2eda60943ea1%2C41643%2C1732387649503.1732387651085, exclude list is [], retry=0 2024-11-23T18:47:31,107 INFO [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346/2eda60943ea1%2C36631%2C1732387649346.1732387651084 2024-11-23T18:47:31,108 DEBUG [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:31,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:31,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:31,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:31,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:31,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:31,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:31,116 INFO [RS:2;2eda60943ea1:41643 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503/2eda60943ea1%2C41643%2C1732387649503.1732387651085 2024-11-23T18:47:31,116 INFO [RS:1;2eda60943ea1:37545 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,37545,1732387649456/2eda60943ea1%2C37545%2C1732387649456.1732387651085 2024-11-23T18:47:31,116 DEBUG [RS:2;2eda60943ea1:41643 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:31,120 DEBUG [RS:1;2eda60943ea1:37545 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:31,170 DEBUG [2eda60943ea1:37311 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-23T18:47:31,177 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T18:47:31,184 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:47:31,184 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:47:31,184 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T18:47:31,184 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:47:31,191 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:31,197 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,41643,1732387649503, state=OPENING 2024-11-23T18:47:31,247 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T18:47:31,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:31,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:31,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:31,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:47:31,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,261 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:47:31,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,41643,1732387649503}] 2024-11-23T18:47:31,446 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:47:31,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38507, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:47:31,458 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:47:31,458 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:31,459 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:47:31,462 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C41643%2C1732387649503.meta, suffix=.meta, logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs, maxLogs=32 2024-11-23T18:47:31,478 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, exclude list is [], retry=0 2024-11-23T18:47:31,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:31,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:31,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:31,485 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta 2024-11-23T18:47:31,486 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:31,486 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:31,488 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:47:31,491 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:47:31,495 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:47:31,499 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:47:31,499 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:31,500 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:47:31,500 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:47:31,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:47:31,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:47:31,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:31,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:31,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:47:31,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:47:31,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:31,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:31,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:47:31,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:47:31,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:31,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:31,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:47:31,511 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:47:31,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:31,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:31,512 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:47:31,514 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740 2024-11-23T18:47:31,517 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740 2024-11-23T18:47:31,519 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:47:31,519 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:47:31,520 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:47:31,523 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:47:31,525 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58935102, jitterRate=-0.1217985451221466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:47:31,525 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:47:31,527 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387651500Writing region info on filesystem at 1732387651500Initializing all the Stores at 1732387651502 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387651502Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387651503 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387651503Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387651503Cleaning up temporary data from old regions at 1732387651519 (+16 ms)Running coprocessor post-open hooks at 1732387651525 (+6 ms)Region opened successfully at 1732387651527 (+2 ms) 2024-11-23T18:47:31,534 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732387651435 2024-11-23T18:47:31,545 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:47:31,546 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:47:31,547 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:31,550 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,41643,1732387649503, state=OPEN 2024-11-23T18:47:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:31,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:31,628 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:31,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T18:47:31,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,41643,1732387649503 in 366 msec 2024-11-23T18:47:31,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T18:47:31,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 886 msec 2024-11-23T18:47:31,646 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T18:47:31,646 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T18:47:31,664 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:31,665 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1] 2024-11-23T18:47:31,685 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:47:31,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59439, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:47:31,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1470 sec 2024-11-23T18:47:31,706 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732387651706, completionTime=-1 2024-11-23T18:47:31,709 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-23T18:47:31,709 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T18:47:31,735 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-23T18:47:31,735 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732387711735 2024-11-23T18:47:31,735 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732387771735 2024-11-23T18:47:31,735 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-23T18:47:31,736 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T18:47:31,742 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37311,1732387648561-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,743 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37311,1732387648561-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,743 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37311,1732387648561-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,744 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2eda60943ea1:37311, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,745 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,745 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,751 DEBUG [master/2eda60943ea1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T18:47:31,772 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.161sec 2024-11-23T18:47:31,774 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T18:47:31,775 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T18:47:31,776 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T18:47:31,777 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T18:47:31,777 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T18:47:31,777 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37311,1732387648561-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T18:47:31,778 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37311,1732387648561-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T18:47:31,783 DEBUG [master/2eda60943ea1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T18:47:31,784 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T18:47:31,784 INFO [master/2eda60943ea1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2eda60943ea1,37311,1732387648561-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:31,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d538c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:47:31,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2eda60943ea1,37311,-1 for getting cluster id 2024-11-23T18:47:31,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T18:47:31,869 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '36796ac2-6f7b-4ddf-9283-66f1c579705e' 2024-11-23T18:47:31,872 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T18:47:31,872 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "36796ac2-6f7b-4ddf-9283-66f1c579705e" 2024-11-23T18:47:31,872 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af201cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:47:31,873 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2eda60943ea1,37311,-1] 2024-11-23T18:47:31,875 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T18:47:31,878 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:47:31,878 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T18:47:31,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a827c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:47:31,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:31,888 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1] 2024-11-23T18:47:31,888 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:47:31,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:47:31,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2eda60943ea1,37311,1732387648561 2024-11-23T18:47:31,909 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:45809/hbase 2024-11-23T18:47:31,924 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=9219 2024-11-23T18:47:31,943 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:31,946 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:31,947 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:31,952 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80665308, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-80665308, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:31,969 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80665308/hregion-80665308.1732387651954, exclude list is [], retry=0 2024-11-23T18:47:31,974 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:31,974 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:31,974 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:31,977 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80665308/hregion-80665308.1732387651954 2024-11-23T18:47:31,980 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:31,980 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b295542b8e39dcaac53132338a37dc52, NAME => 'testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:31,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741838_1014 (size=64) 2024-11-23T18:47:31,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741838_1014 (size=64) 2024-11-23T18:47:31,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741838_1014 (size=64) 2024-11-23T18:47:31,995 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:31,997 INFO [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:31,999 INFO [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b295542b8e39dcaac53132338a37dc52 columnFamilyName a 2024-11-23T18:47:31,999 DEBUG [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,000 INFO [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] regionserver.HStore(327): Store=b295542b8e39dcaac53132338a37dc52/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,000 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,001 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,003 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,003 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,005 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,009 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:32,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b295542b8e39dcaac53132338a37dc52; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65555760, jitterRate=-0.02314305305480957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:32,011 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b295542b8e39dcaac53132338a37dc52: Writing region info on filesystem at 1732387651995Initializing all the Stores at 1732387651996 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387651996Cleaning up temporary data from old regions at 1732387652003 (+7 ms)Region opened successfully at 1732387652011 (+8 ms) 2024-11-23T18:47:32,011 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b295542b8e39dcaac53132338a37dc52, disabling compactions & flushes 2024-11-23T18:47:32,011 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52. 2024-11-23T18:47:32,011 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52. 2024-11-23T18:47:32,011 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52. after waiting 0 ms 2024-11-23T18:47:32,011 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52. 2024-11-23T18:47:32,011 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52. 2024-11-23T18:47:32,012 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b295542b8e39dcaac53132338a37dc52: Waiting for close lock at 1732387652011Disabling compacts and flushes for region at 1732387652011Disabling writes for close at 1732387652011Writing region close event to WAL at 1732387652011Closed at 1732387652011 2024-11-23T18:47:32,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741837_1013 (size=93) 2024-11-23T18:47:32,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741837_1013 (size=93) 2024-11-23T18:47:32,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741837_1013 (size=93) 2024-11-23T18:47:32,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:32,023 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80665308:(num 1732387651954) 2024-11-23T18:47:32,025 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:47:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741839_1015 (size=369) 2024-11-23T18:47:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741839_1015 (size=369) 2024-11-23T18:47:32,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741839_1015 (size=369) 2024-11-23T18:47:32,043 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:47:32,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741840_1016 (size=231) 2024-11-23T18:47:32,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741840_1016 (size=231) 2024-11-23T18:47:32,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741840_1016 (size=231) 2024-11-23T18:47:32,079 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1, size=369 (369bytes) 2024-11-23T18:47:32,079 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-23T18:47:32,080 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-23T18:47:32,080 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1 2024-11-23T18:47:32,085 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1 after 3ms 2024-11-23T18:47:32,091 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1 took 14ms 2024-11-23T18:47:32,095 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1 so closing down 2024-11-23T18:47:32,098 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-23T18:47:32,100 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000001-wal-1.temp 2024-11-23T18:47:32,100 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:32,101 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:32,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741841_1017 (size=369) 2024-11-23T18:47:32,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741841_1017 (size=369) 2024-11-23T18:47:32,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741841_1017 (size=369) 2024-11-23T18:47:32,113 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:32,116 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002 2024-11-23T18:47:32,120 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-23T18:47:32,120 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1, journal: Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1, size=369 (369bytes) at 1732387652079Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1 so closing down at 1732387652095 (+16 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000001-wal-1.temp at 1732387652100 (+5 ms)3 split writer threads finished at 1732387652101 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732387652113 (+12 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002 at 1732387652116 (+3 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732387652120 (+4 ms) 2024-11-23T18:47:32,136 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2, size=231 (231bytes) 2024-11-23T18:47:32,136 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2 2024-11-23T18:47:32,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2 after 1ms 2024-11-23T18:47:32,141 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2 took 6ms 2024-11-23T18:47:32,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2 so closing down 2024-11-23T18:47:32,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:32,146 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-23T18:47:32,148 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002-wal-2.temp 2024-11-23T18:47:32,148 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:32,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741842_1018 (size=231) 2024-11-23T18:47:32,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741842_1018 (size=231) 2024-11-23T18:47:32,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741842_1018 (size=231) 2024-11-23T18:47:32,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:32,165 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-23T18:47:32,167 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-23T18:47:32,167 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2, journal: Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2, size=231 (231bytes) at 1732387652136Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2 so closing down at 1732387652143 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002-wal-2.temp at 1732387652148 (+5 ms)3 split writer threads finished at 1732387652148Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732387652158 (+10 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732387652167 (+9 ms) 2024-11-23T18:47:32,167 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:32,170 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:32,185 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal.1732387652171, exclude list is [], retry=0 2024-11-23T18:47:32,189 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:32,190 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:32,190 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:32,193 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal.1732387652171 2024-11-23T18:47:32,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:32,193 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b295542b8e39dcaac53132338a37dc52, NAME => 'testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:32,193 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:32,193 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,194 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,196 INFO [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,198 INFO [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b295542b8e39dcaac53132338a37dc52 columnFamilyName a 2024-11-23T18:47:32,198 DEBUG [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,200 INFO [StoreOpener-b295542b8e39dcaac53132338a37dc52-1 {}] regionserver.HStore(327): Store=b295542b8e39dcaac53132338a37dc52/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,200 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,201 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,207 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002 2024-11-23T18:47:32,218 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002 2024-11-23T18:47:32,222 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b295542b8e39dcaac53132338a37dc52 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-23T18:47:32,304 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/.tmp/a/ff95a2e0de0f4460af846e803b059c84 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732387652023/Put/seqid=0 2024-11-23T18:47:32,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741844_1020 (size=5170) 2024-11-23T18:47:32,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741844_1020 (size=5170) 2024-11-23T18:47:32,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741844_1020 (size=5170) 2024-11-23T18:47:32,326 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/.tmp/a/ff95a2e0de0f4460af846e803b059c84 2024-11-23T18:47:32,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/.tmp/a/ff95a2e0de0f4460af846e803b059c84 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/a/ff95a2e0de0f4460af846e803b059c84 2024-11-23T18:47:32,380 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/a/ff95a2e0de0f4460af846e803b059c84, entries=2, sequenceid=2, filesize=5.0 K 2024-11-23T18:47:32,385 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b295542b8e39dcaac53132338a37dc52 in 163ms, sequenceid=2, compaction requested=false; wal=null 2024-11-23T18:47:32,387 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/0000000000000000002 2024-11-23T18:47:32,388 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,388 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,392 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b295542b8e39dcaac53132338a37dc52 2024-11-23T18:47:32,396 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/b295542b8e39dcaac53132338a37dc52/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-23T18:47:32,397 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b295542b8e39dcaac53132338a37dc52; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62773677, jitterRate=-0.06459932029247284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:32,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b295542b8e39dcaac53132338a37dc52: Writing region info on filesystem at 1732387652194Initializing all the Stores at 1732387652195 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387652196 (+1 ms)Obtaining lock to block concurrent updates at 1732387652222 (+26 ms)Preparing flush snapshotting stores in b295542b8e39dcaac53132338a37dc52 at 1732387652222Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732387652226 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732387651943.b295542b8e39dcaac53132338a37dc52. at 1732387652226Flushing b295542b8e39dcaac53132338a37dc52/a: creating writer at 1732387652228 (+2 ms)Flushing b295542b8e39dcaac53132338a37dc52/a: appending metadata at 1732387652288 (+60 ms)Flushing b295542b8e39dcaac53132338a37dc52/a: closing flushed file at 1732387652292 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38178104: reopening flushed file at 1732387652367 (+75 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b295542b8e39dcaac53132338a37dc52 in 163ms, sequenceid=2, compaction requested=false; wal=null at 1732387652385 (+18 ms)Cleaning up temporary data from old regions at 1732387652388 (+3 ms)Region opened successfully at 1732387652398 (+10 ms) 2024-11-23T18:47:32,425 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43390 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:45809/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:52844 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:50790 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:34842 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:50806 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43454 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=9129 (was 9219) 2024-11-23T18:47:32,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=9127 2024-11-23T18:47:32,454 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:32,456 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:32,457 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:32,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13042522, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-13042522, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:32,472 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13042522/hregion-13042522.1732387652460, exclude list is [], retry=0 2024-11-23T18:47:32,476 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:32,476 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:32,476 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:32,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13042522/hregion-13042522.1732387652460 2024-11-23T18:47:32,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:32,480 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6e35f396203db399ae9285bc3ef3350e, NAME => 'testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:32,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741846_1022 (size=64) 2024-11-23T18:47:32,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741846_1022 (size=64) 2024-11-23T18:47:32,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741846_1022 (size=64) 2024-11-23T18:47:32,492 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:32,493 INFO [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,496 INFO [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e35f396203db399ae9285bc3ef3350e columnFamilyName a 2024-11-23T18:47:32,496 DEBUG [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,497 INFO [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] regionserver.HStore(327): Store=6e35f396203db399ae9285bc3ef3350e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,498 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,499 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,499 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,500 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,500 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,502 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,505 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:32,506 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6e35f396203db399ae9285bc3ef3350e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69008998, jitterRate=0.028314203023910522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:32,506 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6e35f396203db399ae9285bc3ef3350e: Writing region info on filesystem at 1732387652492Initializing all the Stores at 1732387652493 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387652493Cleaning up temporary data from old regions at 1732387652500 (+7 ms)Region opened successfully at 1732387652506 (+6 ms) 2024-11-23T18:47:32,506 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6e35f396203db399ae9285bc3ef3350e, disabling compactions & flushes 2024-11-23T18:47:32,506 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e. 2024-11-23T18:47:32,506 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e. 2024-11-23T18:47:32,506 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e. after waiting 0 ms 2024-11-23T18:47:32,506 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e. 2024-11-23T18:47:32,507 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e. 2024-11-23T18:47:32,507 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6e35f396203db399ae9285bc3ef3350e: Waiting for close lock at 1732387652506Disabling compacts and flushes for region at 1732387652506Disabling writes for close at 1732387652506Writing region close event to WAL at 1732387652507 (+1 ms)Closed at 1732387652507 2024-11-23T18:47:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741845_1021 (size=93) 2024-11-23T18:47:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741845_1021 (size=93) 2024-11-23T18:47:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741845_1021 (size=93) 2024-11-23T18:47:32,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:32,513 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13042522:(num 1732387652460) 2024-11-23T18:47:32,514 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:47:32,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741847_1023 (size=369) 2024-11-23T18:47:32,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741847_1023 (size=369) 2024-11-23T18:47:32,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741847_1023 (size=369) 2024-11-23T18:47:32,525 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T18:47:32,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741848_1024 (size=231) 2024-11-23T18:47:32,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741848_1024 (size=231) 2024-11-23T18:47:32,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741848_1024 (size=231) 2024-11-23T18:47:32,553 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2, size=231 (231bytes) 2024-11-23T18:47:32,553 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2 2024-11-23T18:47:32,554 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2 after 1ms 2024-11-23T18:47:32,557 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2 took 4ms 2024-11-23T18:47:32,559 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2 so closing down 2024-11-23T18:47:32,560 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:32,562 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-23T18:47:32,563 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002-wal-2.temp 2024-11-23T18:47:32,563 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:32,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741849_1025 (size=231) 2024-11-23T18:47:32,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741849_1025 (size=231) 2024-11-23T18:47:32,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741849_1025 (size=231) 2024-11-23T18:47:32,572 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:32,575 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 2024-11-23T18:47:32,575 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-23T18:47:32,575 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2, journal: Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2, size=231 (231bytes) at 1732387652553Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2 so closing down at 1732387652560 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002-wal-2.temp at 1732387652563 (+3 ms)3 split writer threads finished at 1732387652563Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732387652572 (+9 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 at 1732387652575 (+3 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732387652575 2024-11-23T18:47:32,591 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1, size=369 (369bytes) 2024-11-23T18:47:32,591 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1 2024-11-23T18:47:32,591 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1 after 0ms 2024-11-23T18:47:32,595 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1 took 4ms 2024-11-23T18:47:32,598 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1 so closing down 2024-11-23T18:47:32,598 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:32,601 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-23T18:47:32,602 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000001-wal-1.temp 2024-11-23T18:47:32,603 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:32,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741850_1026 (size=369) 2024-11-23T18:47:32,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741850_1026 (size=369) 2024-11-23T18:47:32,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741850_1026 (size=369) 2024-11-23T18:47:32,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:32,617 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002, length=231 2024-11-23T18:47:32,619 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 2024-11-23T18:47:32,619 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-23T18:47:32,619 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1, journal: Splitting hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1, size=369 (369bytes) at 1732387652591Finishing writing output for hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1 so closing down at 1732387652598 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000001-wal-1.temp at 1732387652602 (+4 ms)3 split writer threads finished at 1732387652603 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732387652611 (+8 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 at 1732387652619 (+8 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732387652619 2024-11-23T18:47:32,620 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:32,622 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:32,638 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal.1732387652623, exclude list is [], retry=0 2024-11-23T18:47:32,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:32,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:32,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:32,645 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal.1732387652623 2024-11-23T18:47:32,645 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:32,645 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e35f396203db399ae9285bc3ef3350e, NAME => 'testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:32,646 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:32,646 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,646 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,648 INFO [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,650 INFO [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e35f396203db399ae9285bc3ef3350e columnFamilyName a 2024-11-23T18:47:32,650 DEBUG [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,651 INFO [StoreOpener-6e35f396203db399ae9285bc3ef3350e-1 {}] regionserver.HStore(327): Store=6e35f396203db399ae9285bc3ef3350e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,655 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 2024-11-23T18:47:32,659 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 2024-11-23T18:47:32,659 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6e35f396203db399ae9285bc3ef3350e 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-23T18:47:32,674 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/.tmp/a/75abe6feb8fa4c91952084114fcf81fb is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732387652513/Put/seqid=0 2024-11-23T18:47:32,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741852_1028 (size=5170) 2024-11-23T18:47:32,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741852_1028 (size=5170) 2024-11-23T18:47:32,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741852_1028 (size=5170) 2024-11-23T18:47:32,684 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/.tmp/a/75abe6feb8fa4c91952084114fcf81fb 2024-11-23T18:47:32,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/.tmp/a/75abe6feb8fa4c91952084114fcf81fb as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/a/75abe6feb8fa4c91952084114fcf81fb 2024-11-23T18:47:32,701 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/a/75abe6feb8fa4c91952084114fcf81fb, entries=2, sequenceid=2, filesize=5.0 K 2024-11-23T18:47:32,701 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6e35f396203db399ae9285bc3ef3350e in 42ms, sequenceid=2, compaction requested=false; wal=null 2024-11-23T18:47:32,702 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/0000000000000000002 2024-11-23T18:47:32,703 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,703 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,707 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6e35f396203db399ae9285bc3ef3350e 2024-11-23T18:47:32,710 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/6e35f396203db399ae9285bc3ef3350e/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-23T18:47:32,712 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6e35f396203db399ae9285bc3ef3350e; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70543342, jitterRate=0.05117771029472351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:32,712 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6e35f396203db399ae9285bc3ef3350e: Writing region info on filesystem at 1732387652646Initializing all the Stores at 1732387652647 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387652647Obtaining lock to block concurrent updates at 1732387652659 (+12 ms)Preparing flush snapshotting stores in 6e35f396203db399ae9285bc3ef3350e at 1732387652659Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732387652659Flushing stores of testReplayEditsWrittenIntoWAL,,1732387652454.6e35f396203db399ae9285bc3ef3350e. at 1732387652659Flushing 6e35f396203db399ae9285bc3ef3350e/a: creating writer at 1732387652659Flushing 6e35f396203db399ae9285bc3ef3350e/a: appending metadata at 1732387652673 (+14 ms)Flushing 6e35f396203db399ae9285bc3ef3350e/a: closing flushed file at 1732387652673Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f0782e: reopening flushed file at 1732387652691 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6e35f396203db399ae9285bc3ef3350e in 42ms, sequenceid=2, compaction requested=false; wal=null at 1732387652701 (+10 ms)Cleaning up temporary data from old regions at 1732387652703 (+2 ms)Region opened successfully at 1732387652712 (+9 ms) 2024-11-23T18:47:32,730 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43390 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43516 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:52844 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:50872 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:50806 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:34920 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=9114 (was 9127) 2024-11-23T18:47:32,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=9113 2024-11-23T18:47:32,755 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:32,757 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:32,758 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:32,762 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25932915, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-25932915, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:32,774 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25932915/hregion-25932915.1732387652762, exclude list is [], retry=0 2024-11-23T18:47:32,778 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:32,778 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:32,779 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:32,781 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25932915/hregion-25932915.1732387652762 2024-11-23T18:47:32,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:32,781 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 97e78793c6be5a49449df293f838f03e, NAME => 'testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:32,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741854_1030 (size=64) 2024-11-23T18:47:32,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741854_1030 (size=64) 2024-11-23T18:47:32,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741854_1030 (size=64) 2024-11-23T18:47:32,793 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:32,795 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,797 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97e78793c6be5a49449df293f838f03e columnFamilyName a 2024-11-23T18:47:32,797 DEBUG [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,798 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(327): Store=97e78793c6be5a49449df293f838f03e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,798 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,800 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97e78793c6be5a49449df293f838f03e columnFamilyName b 2024-11-23T18:47:32,800 DEBUG [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,801 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(327): Store=97e78793c6be5a49449df293f838f03e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,801 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,802 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97e78793c6be5a49449df293f838f03e columnFamilyName c 2024-11-23T18:47:32,802 DEBUG [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:32,803 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(327): Store=97e78793c6be5a49449df293f838f03e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:32,803 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,806 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,806 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,807 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:32,810 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:32,813 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:32,813 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97e78793c6be5a49449df293f838f03e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63571652, jitterRate=-0.052708566188812256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97e78793c6be5a49449df293f838f03e: Writing region info on filesystem at 1732387652794Initializing all the Stores at 1732387652795 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387652795Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387652795Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387652795Cleaning up temporary data from old regions at 1732387652806 (+11 ms)Region opened successfully at 1732387652813 (+7 ms) 2024-11-23T18:47:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97e78793c6be5a49449df293f838f03e, disabling compactions & flushes 2024-11-23T18:47:32,814 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. after waiting 0 ms 2024-11-23T18:47:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:32,814 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97e78793c6be5a49449df293f838f03e: Waiting for close lock at 1732387652814Disabling compacts and flushes for region at 1732387652814Disabling writes for close at 1732387652814Writing region close event to WAL at 1732387652814Closed at 1732387652814 2024-11-23T18:47:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741853_1029 (size=93) 2024-11-23T18:47:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741853_1029 (size=93) 2024-11-23T18:47:32,818 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-25932915/hregion-25932915.1732387652762 not finished, retry = 0 2024-11-23T18:47:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741853_1029 (size=93) 2024-11-23T18:47:32,922 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:32,922 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25932915:(num 1732387652762) 2024-11-23T18:47:32,922 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:32,924 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:32,938 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925, exclude list is [], retry=0 2024-11-23T18:47:32,941 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:32,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:32,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:32,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 2024-11-23T18:47:32,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:33,188 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925, size=0 (0bytes) 2024-11-23T18:47:33,188 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 might be still open, length is 0 2024-11-23T18:47:33,188 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 2024-11-23T18:47:33,189 WARN [IPC Server handler 0 on default port 45809 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-23T18:47:33,190 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 after 2ms 2024-11-23T18:47:33,672 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43550 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39151:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43550 dst: /127.0.0.1:39151 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39151 remote=/127.0.0.1:43550]. Total timeout mills is 60000, 59486 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:33,673 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:34946 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38589:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34946 dst: /127.0.0.1:38589 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:33,673 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:50918 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50918 dst: /127.0.0.1:34245 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:33,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741855_1032 (size=470675) 2024-11-23T18:47:33,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741855_1032 (size=470675) 2024-11-23T18:47:33,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741855_1032 (size=470675) 2024-11-23T18:47:37,017 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:47:37,069 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:47:37,191 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 after 4003ms 2024-11-23T18:47:37,199 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 took 4011ms 2024-11-23T18:47:37,203 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732387652925.temp 2024-11-23T18:47:37,206 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000000001-wal.1732387652925.temp 2024-11-23T18:47:37,273 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925; continuing. 2024-11-23T18:47:37,273 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 so closing down 2024-11-23T18:47:37,273 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:37,273 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:37,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741856_1033 (size=470683) 2024-11-23T18:47:37,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741856_1033 (size=470683) 2024-11-23T18:47:37,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741856_1033 (size=470683) 2024-11-23T18:47:37,280 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000000001-wal.1732387652925.temp (wrote 3002 edits, skipped 0 edits in 27 ms) 2024-11-23T18:47:37,282 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000000001-wal.1732387652925.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000003002 2024-11-23T18:47:37,283 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:47:37,283 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925, journal: Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925, size=0 (0bytes) at 1732387653188Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000000001-wal.1732387652925.temp at 1732387657206 (+4018 ms)Split 1024 edits, skipped 0 edits. at 1732387657245 (+39 ms)Split 2048 edits, skipped 0 edits. at 1732387657259 (+14 ms)Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 so closing down at 1732387657273 (+14 ms)3 split writer threads finished at 1732387657273Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000000001-wal.1732387652925.temp (wrote 3002 edits, skipped 0 edits in 27 ms) at 1732387657281 (+8 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000000001-wal.1732387652925.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000003002 at 1732387657282 (+1 ms)Processed 3002 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925, size=0, length=0, corrupted=false, cancelled=false at 1732387657283 (+1 ms) 2024-11-23T18:47:37,285 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387652925 2024-11-23T18:47:37,287 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000003002 2024-11-23T18:47:37,287 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:37,290 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:37,303 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387657290, exclude list is [], retry=0 2024-11-23T18:47:37,306 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:37,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:37,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:37,310 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387657290 2024-11-23T18:47:37,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:37,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:37,313 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:37,314 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97e78793c6be5a49449df293f838f03e columnFamilyName a 2024-11-23T18:47:37,314 DEBUG [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:37,314 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(327): Store=97e78793c6be5a49449df293f838f03e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:37,315 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:37,316 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97e78793c6be5a49449df293f838f03e columnFamilyName b 2024-11-23T18:47:37,316 DEBUG [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:37,316 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(327): Store=97e78793c6be5a49449df293f838f03e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:37,317 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:37,318 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97e78793c6be5a49449df293f838f03e columnFamilyName c 2024-11-23T18:47:37,318 DEBUG [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:37,318 INFO [StoreOpener-97e78793c6be5a49449df293f838f03e-1 {}] regionserver.HStore(327): Store=97e78793c6be5a49449df293f838f03e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:37,318 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:37,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:37,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:37,322 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000003002 2024-11-23T18:47:37,361 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:47:37,693 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97e78793c6be5a49449df293f838f03e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T18:47:37,733 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/a/71f6ebe2900c4af3b952d99122eea016 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732387652955/Put/seqid=0 2024-11-23T18:47:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741858_1035 (size=50463) 2024-11-23T18:47:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741858_1035 (size=50463) 2024-11-23T18:47:37,742 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/a/71f6ebe2900c4af3b952d99122eea016 2024-11-23T18:47:37,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741858_1035 (size=50463) 2024-11-23T18:47:37,750 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/a/71f6ebe2900c4af3b952d99122eea016 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/a/71f6ebe2900c4af3b952d99122eea016 2024-11-23T18:47:37,757 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/a/71f6ebe2900c4af3b952d99122eea016, entries=754, sequenceid=754, filesize=49.3 K 2024-11-23T18:47:37,758 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 97e78793c6be5a49449df293f838f03e in 65ms, sequenceid=754, compaction requested=false; wal=null 2024-11-23T18:47:37,770 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:47:37,771 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97e78793c6be5a49449df293f838f03e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T18:47:37,780 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/a/601627e94be040f79bcc2529081ef470 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732387653000/Put/seqid=0 2024-11-23T18:47:37,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741859_1036 (size=20072) 2024-11-23T18:47:37,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741859_1036 (size=20072) 2024-11-23T18:47:37,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741859_1036 (size=20072) 2024-11-23T18:47:37,791 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/a/601627e94be040f79bcc2529081ef470 2024-11-23T18:47:37,815 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/b/78d3118a337147c4a51bc9b6e0e5f861 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732387653027/Put/seqid=0 2024-11-23T18:47:37,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741860_1037 (size=35835) 2024-11-23T18:47:37,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741860_1037 (size=35835) 2024-11-23T18:47:37,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741860_1037 (size=35835) 2024-11-23T18:47:37,823 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/b/78d3118a337147c4a51bc9b6e0e5f861 2024-11-23T18:47:37,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/a/601627e94be040f79bcc2529081ef470 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/a/601627e94be040f79bcc2529081ef470 2024-11-23T18:47:37,837 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/a/601627e94be040f79bcc2529081ef470, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-23T18:47:37,839 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/b/78d3118a337147c4a51bc9b6e0e5f861 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/b/78d3118a337147c4a51bc9b6e0e5f861 2024-11-23T18:47:37,848 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/b/78d3118a337147c4a51bc9b6e0e5f861, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-23T18:47:37,849 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 97e78793c6be5a49449df293f838f03e in 77ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-23T18:47:37,866 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:47:37,867 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97e78793c6be5a49449df293f838f03e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T18:47:37,876 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/b/46870bfd5efe477b985471a9475192f5 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732387653054/Put/seqid=0 2024-11-23T18:47:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741861_1038 (size=35082) 2024-11-23T18:47:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741861_1038 (size=35082) 2024-11-23T18:47:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741861_1038 (size=35082) 2024-11-23T18:47:37,885 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/b/46870bfd5efe477b985471a9475192f5 2024-11-23T18:47:37,908 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/c/fe824320c88242708ee426a6cabd5eb1 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732387653096/Put/seqid=0 2024-11-23T18:47:37,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741862_1039 (size=20825) 2024-11-23T18:47:37,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741862_1039 (size=20825) 2024-11-23T18:47:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741862_1039 (size=20825) 2024-11-23T18:47:38,318 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/c/fe824320c88242708ee426a6cabd5eb1 2024-11-23T18:47:38,326 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/b/46870bfd5efe477b985471a9475192f5 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/b/46870bfd5efe477b985471a9475192f5 2024-11-23T18:47:38,334 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/b/46870bfd5efe477b985471a9475192f5, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-23T18:47:38,335 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/c/fe824320c88242708ee426a6cabd5eb1 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/c/fe824320c88242708ee426a6cabd5eb1 2024-11-23T18:47:38,342 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/c/fe824320c88242708ee426a6cabd5eb1, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-23T18:47:38,343 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 97e78793c6be5a49449df293f838f03e in 476ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-23T18:47:38,352 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732387653156/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:38,354 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000003002 2024-11-23T18:47:38,355 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T18:47:38,355 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 97e78793c6be5a49449df293f838f03e 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-23T18:47:38,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/c/dfd2542a3e1b46af8946864e26b06ec7 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732387653107/Put/seqid=0 2024-11-23T18:47:38,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741863_1040 (size=50301) 2024-11-23T18:47:38,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741863_1040 (size=50301) 2024-11-23T18:47:38,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741863_1040 (size=50301) 2024-11-23T18:47:38,377 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/c/dfd2542a3e1b46af8946864e26b06ec7 2024-11-23T18:47:38,384 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dfd2542a3e1b46af8946864e26b06ec7 2024-11-23T18:47:38,385 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/.tmp/c/dfd2542a3e1b46af8946864e26b06ec7 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/c/dfd2542a3e1b46af8946864e26b06ec7 2024-11-23T18:47:38,392 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dfd2542a3e1b46af8946864e26b06ec7 2024-11-23T18:47:38,392 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/c/dfd2542a3e1b46af8946864e26b06ec7, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-23T18:47:38,392 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 97e78793c6be5a49449df293f838f03e in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-23T18:47:38,393 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/0000000000000003002 2024-11-23T18:47:38,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:38,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:38,395 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T18:47:38,397 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 97e78793c6be5a49449df293f838f03e 2024-11-23T18:47:38,400 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenIntoWAL/97e78793c6be5a49449df293f838f03e/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-23T18:47:38,401 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 97e78793c6be5a49449df293f838f03e; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62762254, jitterRate=-0.06476953625679016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T18:47:38,401 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 97e78793c6be5a49449df293f838f03e: Writing region info on filesystem at 1732387657311Initializing all the Stores at 1732387657312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387657312Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387657312Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387657312Cleaning up temporary data from old regions at 1732387658394 (+1082 ms)Region opened successfully at 1732387658401 (+7 ms) 2024-11-23T18:47:38,467 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 97e78793c6be5a49449df293f838f03e, disabling compactions & flushes 2024-11-23T18:47:38,467 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:38,467 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:38,467 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. after waiting 0 ms 2024-11-23T18:47:38,467 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:38,469 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732387652756.97e78793c6be5a49449df293f838f03e. 2024-11-23T18:47:38,469 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 97e78793c6be5a49449df293f838f03e: Waiting for close lock at 1732387658466Disabling compacts and flushes for region at 1732387658466Disabling writes for close at 1732387658467 (+1 ms)Writing region close event to WAL at 1732387658469 (+2 ms)Closed at 1732387658469 2024-11-23T18:47:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741857_1034 (size=93) 2024-11-23T18:47:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741857_1034 (size=93) 2024-11-23T18:47:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741857_1034 (size=93) 2024-11-23T18:47:38,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:38,475 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387657290) 2024-11-23T18:47:38,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:45809 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45623 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_55954127_22 at /127.0.0.1:34958 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36491 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_55954127_22 at /127.0.0.1:50936 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:36491 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:45809 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:45623 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_55954127_22 at /127.0.0.1:43592 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_55954127_22 at /127.0.0.1:43620 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=9049 (was 9113) 2024-11-23T18:47:38,500 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=9048 2024-11-23T18:47:38,516 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:38,518 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:38,519 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:38,522 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41712605, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-41712605, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:38,535 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41712605/hregion-41712605.1732387658523, exclude list is [], retry=0 2024-11-23T18:47:38,538 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:38,539 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:38,539 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:38,542 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41712605/hregion-41712605.1732387658523 2024-11-23T18:47:38,542 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:38,542 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c209803a682a2c09deda39fa5b6e6e39, NAME => 'test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741865_1042 (size=43) 2024-11-23T18:47:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741865_1042 (size=43) 2024-11-23T18:47:38,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741865_1042 (size=43) 2024-11-23T18:47:38,559 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:38,561 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,563 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c209803a682a2c09deda39fa5b6e6e39 columnFamilyName a 2024-11-23T18:47:38,563 DEBUG [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:38,563 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(327): Store=c209803a682a2c09deda39fa5b6e6e39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:38,563 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,565 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c209803a682a2c09deda39fa5b6e6e39 columnFamilyName b 2024-11-23T18:47:38,565 DEBUG [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:38,566 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(327): Store=c209803a682a2c09deda39fa5b6e6e39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:38,566 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,568 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c209803a682a2c09deda39fa5b6e6e39 columnFamilyName c 2024-11-23T18:47:38,568 DEBUG [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:38,568 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(327): Store=c209803a682a2c09deda39fa5b6e6e39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:38,569 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,571 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,571 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,572 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:38,573 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:38,575 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:38,576 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c209803a682a2c09deda39fa5b6e6e39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74609532, jitterRate=0.11176866292953491}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c209803a682a2c09deda39fa5b6e6e39: Writing region info on filesystem at 1732387658559Initializing all the Stores at 1732387658560 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387658560Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387658561 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387658561Cleaning up temporary data from old regions at 1732387658571 (+10 ms)Region opened successfully at 1732387658576 (+5 ms) 2024-11-23T18:47:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c209803a682a2c09deda39fa5b6e6e39, disabling compactions & flushes 2024-11-23T18:47:38,577 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. after waiting 0 ms 2024-11-23T18:47:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:38,577 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:38,577 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c209803a682a2c09deda39fa5b6e6e39: Waiting for close lock at 1732387658577Disabling compacts and flushes for region at 1732387658577Disabling writes for close at 1732387658577Writing region close event to WAL at 1732387658577Closed at 1732387658577 2024-11-23T18:47:38,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741864_1041 (size=93) 2024-11-23T18:47:38,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741864_1041 (size=93) 2024-11-23T18:47:38,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741864_1041 (size=93) 2024-11-23T18:47:38,583 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:38,583 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41712605:(num 1732387658523) 2024-11-23T18:47:38,583 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:38,585 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:38,598 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586, exclude list is [], retry=0 2024-11-23T18:47:38,601 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:38,601 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:38,601 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:38,603 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 2024-11-23T18:47:38,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741866_1043 (size=344306) 2024-11-23T18:47:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741866_1043 (size=344306) 2024-11-23T18:47:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741866_1043 (size=344306) 2024-11-23T18:47:38,749 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586, size=336.2 K (344306bytes) 2024-11-23T18:47:38,749 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 2024-11-23T18:47:38,750 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 after 1ms 2024-11-23T18:47:38,754 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 took 5ms 2024-11-23T18:47:38,758 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732387658586.temp 2024-11-23T18:47:38,759 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000000001-wal.1732387658586.temp 2024-11-23T18:47:38,778 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 so closing down 2024-11-23T18:47:38,778 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:38,785 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:38,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741867_1044 (size=344306) 2024-11-23T18:47:38,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741867_1044 (size=344306) 2024-11-23T18:47:38,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741867_1044 (size=344306) 2024-11-23T18:47:38,790 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000000001-wal.1732387658586.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-23T18:47:38,792 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000000001-wal.1732387658586.temp to hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003000 2024-11-23T18:47:38,792 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-23T18:47:38,793 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586, journal: Splitting hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586, size=336.2 K (344306bytes) at 1732387658749Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000000001-wal.1732387658586.temp at 1732387658759 (+10 ms)Split 1024 edits, skipped 0 edits. at 1732387658763 (+4 ms)Split 2048 edits, skipped 0 edits. at 1732387658771 (+8 ms)Finishing writing output for hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 so closing down at 1732387658778 (+7 ms)3 split writer threads finished at 1732387658785 (+7 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000000001-wal.1732387658586.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1732387658790 (+5 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000000001-wal.1732387658586.temp to hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003000 at 1732387658792 (+2 ms)Processed 3000 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732387658792 2024-11-23T18:47:38,795 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658586 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387658586 2024-11-23T18:47:38,796 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003000 2024-11-23T18:47:38,796 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:38,799 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:38,819 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800, exclude list is [], retry=0 2024-11-23T18:47:38,822 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:38,822 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:38,823 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:38,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 2024-11-23T18:47:38,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:38,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T18:47:38,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:38,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:47:38,897 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T18:47:38,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T18:47:38,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:38,899 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-23T18:47:38,899 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741868_1045 (size=344433) 2024-11-23T18:47:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741868_1045 (size=344433) 2024-11-23T18:47:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741868_1045 (size=344433) 2024-11-23T18:47:38,977 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800, size=336.4 K (344433bytes) 2024-11-23T18:47:38,977 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 2024-11-23T18:47:38,977 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 after 0ms 2024-11-23T18:47:38,982 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 took 5ms 2024-11-23T18:47:38,986 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732387658800.temp 2024-11-23T18:47:38,987 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003001-wal.1732387658800.temp 2024-11-23T18:47:39,012 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 so closing down 2024-11-23T18:47:39,012 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:39,013 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741869_1046 (size=344433) 2024-11-23T18:47:39,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741869_1046 (size=344433) 2024-11-23T18:47:39,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741869_1046 (size=344433) 2024-11-23T18:47:39,017 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003001-wal.1732387658800.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-23T18:47:39,018 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003001-wal.1732387658800.temp to hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000006000 2024-11-23T18:47:39,019 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-23T18:47:39,019 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800, journal: Splitting hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800, size=336.4 K (344433bytes) at 1732387658977Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003001-wal.1732387658800.temp at 1732387658987 (+10 ms)Split 1024 edits, skipped 0 edits. at 1732387658994 (+7 ms)Split 2048 edits, skipped 0 edits. at 1732387659004 (+10 ms)Finishing writing output for hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 so closing down at 1732387659012 (+8 ms)3 split writer threads finished at 1732387659013 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003001-wal.1732387658800.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1732387659017 (+4 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003001-wal.1732387658800.temp to hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000006000 at 1732387659018 (+1 ms)Processed 3000 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732387659019 (+1 ms) 2024-11-23T18:47:39,021 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387658800 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387658800 2024-11-23T18:47:39,022 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000006000 2024-11-23T18:47:39,022 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:39,024 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/test2727-manual,16010,1732387658515, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:39,038 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387659024, exclude list is [], retry=0 2024-11-23T18:47:39,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:39,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:39,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:39,043 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732387658515/wal.1732387659024 2024-11-23T18:47:39,044 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:39,044 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c209803a682a2c09deda39fa5b6e6e39, NAME => 'test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:39,044 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:39,044 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,045 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,046 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,048 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c209803a682a2c09deda39fa5b6e6e39 columnFamilyName a 2024-11-23T18:47:39,048 DEBUG [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,048 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(327): Store=c209803a682a2c09deda39fa5b6e6e39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,049 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,050 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c209803a682a2c09deda39fa5b6e6e39 columnFamilyName b 2024-11-23T18:47:39,050 DEBUG [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,050 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(327): Store=c209803a682a2c09deda39fa5b6e6e39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,051 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,052 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c209803a682a2c09deda39fa5b6e6e39 columnFamilyName c 2024-11-23T18:47:39,052 DEBUG [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,053 INFO [StoreOpener-c209803a682a2c09deda39fa5b6e6e39-1 {}] regionserver.HStore(327): Store=c209803a682a2c09deda39fa5b6e6e39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,053 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,054 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,056 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,057 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003000 2024-11-23T18:47:39,090 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003000 2024-11-23T18:47:39,091 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000006000 2024-11-23T18:47:39,126 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000006000 2024-11-23T18:47:39,126 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c209803a682a2c09deda39fa5b6e6e39 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-23T18:47:39,148 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/a/0e45f7f4a33347fb8ab6c786f9d8e96d is 41, key is test2727/a:100/1732387658830/Put/seqid=0 2024-11-23T18:47:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741871_1048 (size=84227) 2024-11-23T18:47:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741871_1048 (size=84227) 2024-11-23T18:47:39,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741871_1048 (size=84227) 2024-11-23T18:47:39,155 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/a/0e45f7f4a33347fb8ab6c786f9d8e96d 2024-11-23T18:47:39,191 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/b/09c91cf86dfc4095bbf2c548222f017a is 41, key is test2727/b:100/1732387658868/Put/seqid=0 2024-11-23T18:47:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741872_1049 (size=84609) 2024-11-23T18:47:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741872_1049 (size=84609) 2024-11-23T18:47:39,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741872_1049 (size=84609) 2024-11-23T18:47:39,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/b/09c91cf86dfc4095bbf2c548222f017a 2024-11-23T18:47:39,232 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/c/0badf5d338554da49726ee5e0c219a22 is 41, key is test2727/c:100/1732387658907/Put/seqid=0 2024-11-23T18:47:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741873_1050 (size=84609) 2024-11-23T18:47:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741873_1050 (size=84609) 2024-11-23T18:47:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741873_1050 (size=84609) 2024-11-23T18:47:39,239 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/c/0badf5d338554da49726ee5e0c219a22 2024-11-23T18:47:39,247 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/a/0e45f7f4a33347fb8ab6c786f9d8e96d as hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/a/0e45f7f4a33347fb8ab6c786f9d8e96d 2024-11-23T18:47:39,256 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/a/0e45f7f4a33347fb8ab6c786f9d8e96d, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-23T18:47:39,258 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/b/09c91cf86dfc4095bbf2c548222f017a as hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/b/09c91cf86dfc4095bbf2c548222f017a 2024-11-23T18:47:39,265 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/b/09c91cf86dfc4095bbf2c548222f017a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-23T18:47:39,266 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/.tmp/c/0badf5d338554da49726ee5e0c219a22 as hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/c/0badf5d338554da49726ee5e0c219a22 2024-11-23T18:47:39,273 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/c/0badf5d338554da49726ee5e0c219a22, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-23T18:47:39,273 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for c209803a682a2c09deda39fa5b6e6e39 in 147ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-23T18:47:39,274 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000003000 2024-11-23T18:47:39,275 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/0000000000000006000 2024-11-23T18:47:39,276 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,276 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,277 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:39,279 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c209803a682a2c09deda39fa5b6e6e39 2024-11-23T18:47:39,281 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/test2727/c209803a682a2c09deda39fa5b6e6e39/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-23T18:47:39,283 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c209803a682a2c09deda39fa5b6e6e39; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67384990, jitterRate=0.004114598035812378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:39,284 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c209803a682a2c09deda39fa5b6e6e39: Writing region info on filesystem at 1732387659045Initializing all the Stores at 1732387659046 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659046Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659046Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659046Obtaining lock to block concurrent updates at 1732387659126 (+80 ms)Preparing flush snapshotting stores in c209803a682a2c09deda39fa5b6e6e39 at 1732387659126Finished memstore snapshotting test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732387659126Flushing stores of test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. at 1732387659126Flushing c209803a682a2c09deda39fa5b6e6e39/a: creating writer at 1732387659126Flushing c209803a682a2c09deda39fa5b6e6e39/a: appending metadata at 1732387659147 (+21 ms)Flushing c209803a682a2c09deda39fa5b6e6e39/a: closing flushed file at 1732387659147Flushing c209803a682a2c09deda39fa5b6e6e39/b: creating writer at 1732387659162 (+15 ms)Flushing c209803a682a2c09deda39fa5b6e6e39/b: appending metadata at 1732387659189 (+27 ms)Flushing c209803a682a2c09deda39fa5b6e6e39/b: closing flushed file at 1732387659189Flushing c209803a682a2c09deda39fa5b6e6e39/c: creating writer at 1732387659203 (+14 ms)Flushing c209803a682a2c09deda39fa5b6e6e39/c: appending metadata at 1732387659231 (+28 ms)Flushing c209803a682a2c09deda39fa5b6e6e39/c: closing flushed file at 1732387659231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d6927fc: reopening flushed file at 1732387659246 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58081c83: reopening flushed file at 1732387659256 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cc194c9: reopening flushed file at 1732387659265 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for c209803a682a2c09deda39fa5b6e6e39 in 147ms, sequenceid=6000, compaction requested=false; wal=null at 1732387659273 (+8 ms)Cleaning up temporary data from old regions at 1732387659276 (+3 ms)Region opened successfully at 1732387659283 (+7 ms) 2024-11-23T18:47:39,285 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-23T18:47:39,285 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c209803a682a2c09deda39fa5b6e6e39, disabling compactions & flushes 2024-11-23T18:47:39,285 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:39,285 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:39,285 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. after waiting 0 ms 2024-11-23T18:47:39,285 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:39,286 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732387658517.c209803a682a2c09deda39fa5b6e6e39. 2024-11-23T18:47:39,286 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c209803a682a2c09deda39fa5b6e6e39: Waiting for close lock at 1732387659285Disabling compacts and flushes for region at 1732387659285Disabling writes for close at 1732387659285Writing region close event to WAL at 1732387659286 (+1 ms)Closed at 1732387659286 2024-11-23T18:47:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741870_1047 (size=93) 2024-11-23T18:47:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741870_1047 (size=93) 2024-11-23T18:47:39,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741870_1047 (size=93) 2024-11-23T18:47:39,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:39,292 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387659024) 2024-11-23T18:47:39,304 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=403 (was 400) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:51054 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43592 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:35106 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43620 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=8923 (was 9048) 2024-11-23T18:47:39,315 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=8922 2024-11-23T18:47:39,330 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:39,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:39,338 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732387659338 2024-11-23T18:47:39,345 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338 2024-11-23T18:47:39,352 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:39,356 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d172cfaa30610a91ff8312567fb0a12b, NAME => 'testSequentialEditLogSeqNum,,1732387659331.d172cfaa30610a91ff8312567fb0a12b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:39,356 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732387659331.d172cfaa30610a91ff8312567fb0a12b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:39,356 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,356 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,357 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b doesn't exist for region: d172cfaa30610a91ff8312567fb0a12b on table testSequentialEditLogSeqNum 2024-11-23T18:47:39,358 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d172cfaa30610a91ff8312567fb0a12b on table testSequentialEditLogSeqNum 2024-11-23T18:47:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741875_1052 (size=62) 2024-11-23T18:47:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741875_1052 (size=62) 2024-11-23T18:47:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741875_1052 (size=62) 2024-11-23T18:47:39,371 INFO [StoreOpener-d172cfaa30610a91ff8312567fb0a12b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,373 INFO [StoreOpener-d172cfaa30610a91ff8312567fb0a12b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d172cfaa30610a91ff8312567fb0a12b columnFamilyName a 2024-11-23T18:47:39,373 DEBUG [StoreOpener-d172cfaa30610a91ff8312567fb0a12b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,374 INFO [StoreOpener-d172cfaa30610a91ff8312567fb0a12b-1 {}] regionserver.HStore(327): Store=d172cfaa30610a91ff8312567fb0a12b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,374 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,376 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,376 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d172cfaa30610a91ff8312567fb0a12b 2024-11-23T18:47:39,380 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:39,381 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d172cfaa30610a91ff8312567fb0a12b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70362518, jitterRate=0.0484832227230072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:39,382 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d172cfaa30610a91ff8312567fb0a12b: Writing region info on filesystem at 1732387659356Initializing all the Stores at 1732387659371 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659371Cleaning up temporary data from old regions at 1732387659376 (+5 ms)Region opened successfully at 1732387659382 (+6 ms) 2024-11-23T18:47:39,393 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d172cfaa30610a91ff8312567fb0a12b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-23T18:47:39,412 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/.tmp/a/db2728ff73f04a05a832530aabbf4c14 is 81, key is testSequentialEditLogSeqNum/a:x0/1732387659382/Put/seqid=0 2024-11-23T18:47:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741876_1053 (size=5833) 2024-11-23T18:47:39,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741876_1053 (size=5833) 2024-11-23T18:47:39,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741876_1053 (size=5833) 2024-11-23T18:47:39,419 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/.tmp/a/db2728ff73f04a05a832530aabbf4c14 2024-11-23T18:47:39,427 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/.tmp/a/db2728ff73f04a05a832530aabbf4c14 as hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/a/db2728ff73f04a05a832530aabbf4c14 2024-11-23T18:47:39,434 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/a/db2728ff73f04a05a832530aabbf4c14, entries=10, sequenceid=13, filesize=5.7 K 2024-11-23T18:47:39,435 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d172cfaa30610a91ff8312567fb0a12b in 43ms, sequenceid=13, compaction requested=false 2024-11-23T18:47:39,435 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d172cfaa30610a91ff8312567fb0a12b: 2024-11-23T18:47:39,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:47:39,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:47:39,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:47:39,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:47:39,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T18:47:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741874_1051 (size=3017) 2024-11-23T18:47:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741874_1051 (size=3017) 2024-11-23T18:47:39,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741874_1051 (size=3017) 2024-11-23T18:47:39,460 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338, size=2.9 K (3017bytes) 2024-11-23T18:47:39,460 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338 2024-11-23T18:47:39,460 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338 after 0ms 2024-11-23T18:47:39,463 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338 took 3ms 2024-11-23T18:47:39,466 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338 so closing down 2024-11-23T18:47:39,466 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:39,467 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387659338.temp 2024-11-23T18:47:39,468 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000003-wal.1732387659338.temp 2024-11-23T18:47:39,468 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:39,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741877_1054 (size=2433) 2024-11-23T18:47:39,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741877_1054 (size=2433) 2024-11-23T18:47:39,478 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000003-wal.1732387659338.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:39,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741877_1054 (size=2433) 2024-11-23T18:47:39,479 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000003-wal.1732387659338.temp to hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000020 2024-11-23T18:47:39,479 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-23T18:47:39,479 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338, journal: Splitting hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338, size=2.9 K (3017bytes) at 1732387659460Finishing writing output for hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338 so closing down at 1732387659466 (+6 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000003-wal.1732387659338.temp at 1732387659468 (+2 ms)3 split writer threads finished at 1732387659469 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000003-wal.1732387659338.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732387659478 (+9 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000003-wal.1732387659338.temp to hdfs://localhost:45809/hbase/data/default/testSequentialEditLogSeqNum/d172cfaa30610a91ff8312567fb0a12b/recovered.edits/0000000000000000020 at 1732387659479 (+1 ms)Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45809/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732387659329/wal.1732387659338, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732387659479 2024-11-23T18:47:39,492 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:51054 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43592 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43620 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=8910 (was 8922) 2024-11-23T18:47:39,502 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=8909 2024-11-23T18:47:39,517 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:39,520 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:39,553 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:39,556 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85112725, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-85112725, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:39,575 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85112725/hregion-85112725.1732387659557, exclude list is [], retry=0 2024-11-23T18:47:39,579 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:39,579 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:39,580 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:39,582 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85112725/hregion-85112725.1732387659557 2024-11-23T18:47:39,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:39,585 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a4df809d520ace41e0b5d41a9288fa7b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:39,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741879_1056 (size=70) 2024-11-23T18:47:39,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741879_1056 (size=70) 2024-11-23T18:47:39,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741879_1056 (size=70) 2024-11-23T18:47:39,595 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:39,597 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,599 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName a 2024-11-23T18:47:39,599 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,599 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,599 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,601 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName b 2024-11-23T18:47:39,601 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,602 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,602 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,604 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName c 2024-11-23T18:47:39,604 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,604 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,605 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,607 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,607 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,608 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:39,609 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,611 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:39,612 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4df809d520ace41e0b5d41a9288fa7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64023399, jitterRate=-0.04597701132297516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:39,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4df809d520ace41e0b5d41a9288fa7b: Writing region info on filesystem at 1732387659595Initializing all the Stores at 1732387659596 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659597 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659597Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659597Cleaning up temporary data from old regions at 1732387659607 (+10 ms)Region opened successfully at 1732387659613 (+6 ms) 2024-11-23T18:47:39,613 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4df809d520ace41e0b5d41a9288fa7b, disabling compactions & flushes 2024-11-23T18:47:39,613 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:39,613 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:39,613 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. after waiting 0 ms 2024-11-23T18:47:39,613 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:39,614 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:39,614 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4df809d520ace41e0b5d41a9288fa7b: Waiting for close lock at 1732387659613Disabling compacts and flushes for region at 1732387659613Disabling writes for close at 1732387659613Writing region close event to WAL at 1732387659614 (+1 ms)Closed at 1732387659614 2024-11-23T18:47:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741878_1055 (size=93) 2024-11-23T18:47:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741878_1055 (size=93) 2024-11-23T18:47:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741878_1055 (size=93) 2024-11-23T18:47:39,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:39,620 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85112725:(num 1732387659557) 2024-11-23T18:47:39,620 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:39,623 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:39,638 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624, exclude list is [], retry=0 2024-11-23T18:47:39,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:39,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:39,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:39,644 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 2024-11-23T18:47:39,645 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:39,645 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4df809d520ace41e0b5d41a9288fa7b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:39,645 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:39,645 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,646 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,647 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,649 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName a 2024-11-23T18:47:39,649 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,650 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,650 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,651 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName b 2024-11-23T18:47:39,651 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,652 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,652 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,653 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName c 2024-11-23T18:47:39,653 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:39,654 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:39,654 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,655 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,656 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,658 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,658 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,658 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:39,660 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:39,661 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4df809d520ace41e0b5d41a9288fa7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68232876, jitterRate=0.016749083995819092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:39,663 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4df809d520ace41e0b5d41a9288fa7b: Writing region info on filesystem at 1732387659646Initializing all the Stores at 1732387659647 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659647Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659647Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387659647Cleaning up temporary data from old regions at 1732387659658 (+11 ms)Region opened successfully at 1732387659663 (+5 ms) 2024-11-23T18:47:39,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732387659667/Put/seqid=0 2024-11-23T18:47:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741881_1058 (size=4826) 2024-11-23T18:47:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741881_1058 (size=4826) 2024-11-23T18:47:39,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741881_1058 (size=4826) 2024-11-23T18:47:39,676 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45809/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in a4df809d520ace41e0b5d41a9288fa7b/a 2024-11-23T18:47:39,684 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-23T18:47:39,684 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:47:39,685 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a4df809d520ace41e0b5d41a9288fa7b: 2024-11-23T18:47:39,687 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_ 2024-11-23T18:47:39,687 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45809/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a4df809d520ace41e0b5d41a9288fa7b/a as hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_ - updating store file list. 2024-11-23T18:47:39,693 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-23T18:47:39,694 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_ into a4df809d520ace41e0b5d41a9288fa7b/a 2024-11-23T18:47:39,694 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45809/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a4df809d520ace41e0b5d41a9288fa7b/a (new location: hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_) 2024-11-23T18:47:39,738 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624, size=0 (0bytes) 2024-11-23T18:47:39,738 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 might be still open, length is 0 2024-11-23T18:47:39,738 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 2024-11-23T18:47:39,738 WARN [IPC Server handler 1 on default port 45809 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-23T18:47:39,739 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 after 0ms 2024-11-23T18:47:42,666 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:43808 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39151:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43808 dst: /127.0.0.1:39151 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39151 remote=/127.0.0.1:43808]. Total timeout mills is 60000, 57033 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:42,667 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:51118 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51118 dst: /127.0.0.1:34245 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:42,667 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:35200 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38589:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35200 dst: /127.0.0.1:38589 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:42,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741880_1059 (size=504) 2024-11-23T18:47:42,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741880_1059 (size=504) 2024-11-23T18:47:43,740 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 after 4001ms 2024-11-23T18:47:43,745 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 took 4008ms 2024-11-23T18:47:43,749 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624; continuing. 2024-11-23T18:47:43,749 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 so closing down 2024-11-23T18:47:43,749 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:43,751 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732387659624.temp 2024-11-23T18:47:43,753 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005-wal.1732387659624.temp 2024-11-23T18:47:43,753 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741882_1060 (size=238) 2024-11-23T18:47:43,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741882_1060 (size=238) 2024-11-23T18:47:43,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741882_1060 (size=238) 2024-11-23T18:47:43,761 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005-wal.1732387659624.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:43,762 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005-wal.1732387659624.temp to hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005 2024-11-23T18:47:43,762 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:47:43,762 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624, journal: Splitting hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624, size=0 (0bytes) at 1732387659738Finishing writing output for hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 so closing down at 1732387663749 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005-wal.1732387659624.temp at 1732387663753 (+4 ms)3 split writer threads finished at 1732387663753Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005-wal.1732387659624.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732387663761 (+8 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005-wal.1732387659624.temp to hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005 at 1732387663762 (+1 ms)Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624, size=0, length=0, corrupted=false, cancelled=false at 1732387663762 2024-11-23T18:47:43,764 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387659624 2024-11-23T18:47:43,765 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005 2024-11-23T18:47:43,765 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:43,767 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:43,782 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387663767, exclude list is [], retry=0 2024-11-23T18:47:43,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:43,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:43,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:43,787 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387663767 2024-11-23T18:47:43,787 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:43,787 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4df809d520ace41e0b5d41a9288fa7b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:43,788 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:43,788 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,788 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,789 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,790 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName a 2024-11-23T18:47:43,790 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:43,797 DEBUG [StoreFileOpener-a4df809d520ace41e0b5d41a9288fa7b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-23T18:47:43,797 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/2a26d73eceed4ea0ba6a4cbf4803c12b_SeqId_3_ 2024-11-23T18:47:43,797 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:43,798 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,799 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName b 2024-11-23T18:47:43,799 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:43,799 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:43,799 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,800 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4df809d520ace41e0b5d41a9288fa7b columnFamilyName c 2024-11-23T18:47:43,800 DEBUG [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:43,801 INFO [StoreOpener-a4df809d520ace41e0b5d41a9288fa7b-1 {}] regionserver.HStore(327): Store=a4df809d520ace41e0b5d41a9288fa7b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:43,801 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,802 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,804 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005 2024-11-23T18:47:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005 2024-11-23T18:47:43,808 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4df809d520ace41e0b5d41a9288fa7b 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-23T18:47:43,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/.tmp/a/dfb66b2b44564a379e1f637c8f8fab88 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732387659699/Put/seqid=0 2024-11-23T18:47:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741884_1062 (size=5149) 2024-11-23T18:47:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741884_1062 (size=5149) 2024-11-23T18:47:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741884_1062 (size=5149) 2024-11-23T18:47:43,829 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/.tmp/a/dfb66b2b44564a379e1f637c8f8fab88 2024-11-23T18:47:43,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/.tmp/a/dfb66b2b44564a379e1f637c8f8fab88 as hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/dfb66b2b44564a379e1f637c8f8fab88 2024-11-23T18:47:43,843 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/a/dfb66b2b44564a379e1f637c8f8fab88, entries=1, sequenceid=5, filesize=5.0 K 2024-11-23T18:47:43,843 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a4df809d520ace41e0b5d41a9288fa7b in 35ms, sequenceid=5, compaction requested=false; wal=null 2024-11-23T18:47:43,844 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/0000000000000000005 2024-11-23T18:47:43,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,846 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:43,848 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4df809d520ace41e0b5d41a9288fa7b 2024-11-23T18:47:43,850 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a4df809d520ace41e0b5d41a9288fa7b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-23T18:47:43,851 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4df809d520ace41e0b5d41a9288fa7b; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69364968, jitterRate=0.03361856937408447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:43,852 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4df809d520ace41e0b5d41a9288fa7b: Writing region info on filesystem at 1732387663788Initializing all the Stores at 1732387663789 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387663789Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387663789Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387663789Obtaining lock to block concurrent updates at 1732387663808 (+19 ms)Preparing flush snapshotting stores in a4df809d520ace41e0b5d41a9288fa7b at 1732387663808Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732387663808Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. at 1732387663808Flushing a4df809d520ace41e0b5d41a9288fa7b/a: creating writer at 1732387663808Flushing a4df809d520ace41e0b5d41a9288fa7b/a: appending metadata at 1732387663822 (+14 ms)Flushing a4df809d520ace41e0b5d41a9288fa7b/a: closing flushed file at 1732387663822Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bfeea1a: reopening flushed file at 1732387663835 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a4df809d520ace41e0b5d41a9288fa7b in 35ms, sequenceid=5, compaction requested=false; wal=null at 1732387663843 (+8 ms)Cleaning up temporary data from old regions at 1732387663845 (+2 ms)Region opened successfully at 1732387663852 (+7 ms) 2024-11-23T18:47:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4df809d520ace41e0b5d41a9288fa7b, disabling compactions & flushes 2024-11-23T18:47:43,857 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. after waiting 0 ms 2024-11-23T18:47:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:43,858 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732387659518.a4df809d520ace41e0b5d41a9288fa7b. 2024-11-23T18:47:43,858 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4df809d520ace41e0b5d41a9288fa7b: Waiting for close lock at 1732387663857Disabling compacts and flushes for region at 1732387663857Disabling writes for close at 1732387663857Writing region close event to WAL at 1732387663858 (+1 ms)Closed at 1732387663858 2024-11-23T18:47:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741883_1061 (size=93) 2024-11-23T18:47:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741883_1061 (size=93) 2024-11-23T18:47:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741883_1061 (size=93) 2024-11-23T18:47:43,863 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:43,863 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387663767) 2024-11-23T18:47:43,875 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 408) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:45809 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:45809 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_254575852_22 at /127.0.0.1:51022 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_254575852_22 at /127.0.0.1:55934 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_254575852_22 at /127.0.0.1:58452 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=8863 (was 8909) 2024-11-23T18:47:43,876 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-23T18:47:43,886 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=8862 2024-11-23T18:47:43,886 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-23T18:47:43,902 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:43,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T18:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2eda60943ea1,37311,1732387648561 2024-11-23T18:47:43,912 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@674a5408 2024-11-23T18:47:43,913 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T18:47:43,915 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T18:47:43,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T18:47:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-23T18:47:43,926 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T18:47:43,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-23T18:47:43,929 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:43,930 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T18:47:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:47:43,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741885_1063 (size=694) 2024-11-23T18:47:43,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741885_1063 (size=694) 2024-11-23T18:47:43,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741885_1063 (size=694) 2024-11-23T18:47:43,943 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f 2024-11-23T18:47:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741886_1064 (size=77) 2024-11-23T18:47:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741886_1064 (size=77) 2024-11-23T18:47:43,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741886_1064 (size=77) 2024-11-23T18:47:43,952 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:43,952 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:47:43,952 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:43,952 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:43,952 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:47:43,952 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:43,952 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:43,952 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387663952Disabling compacts and flushes for region at 1732387663952Disabling writes for close at 1732387663952Writing region close event to WAL at 1732387663952Closed at 1732387663952 2024-11-23T18:47:43,954 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T18:47:43,959 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732387663954"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732387663954"}]},"ts":"1732387663954"} 2024-11-23T18:47:43,962 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T18:47:43,964 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T18:47:43,966 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387663964"}]},"ts":"1732387663964"} 2024-11-23T18:47:43,969 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-23T18:47:43,970 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:47:43,971 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:47:43,971 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:47:43,971 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T18:47:43,971 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:47:43,971 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:47:43,971 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T18:47:43,971 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:47:43,972 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:47:43,972 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T18:47:43,972 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:47:43,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN}] 2024-11-23T18:47:43,975 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN 2024-11-23T18:47:43,976 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN; state=OFFLINE, location=2eda60943ea1,41643,1732387649503; forceNewPlan=false, retain=false 2024-11-23T18:47:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:47:44,132 INFO [2eda60943ea1:37311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T18:47:44,133 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPENING, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:44,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN because future has completed 2024-11-23T18:47:44,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503}] 2024-11-23T18:47:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:47:44,297 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,297 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:44,298 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,298 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:44,298 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,298 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,300 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,302 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf1 2024-11-23T18:47:44,302 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:44,302 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:44,303 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,304 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf2 2024-11-23T18:47:44,304 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:44,305 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:44,305 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,307 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,308 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,309 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,309 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,310 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:47:44,311 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,314 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:44,315 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e2564e420fa6e11419482705c96414e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65652081, jitterRate=-0.02170775830745697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:47:44,315 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,316 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e2564e420fa6e11419482705c96414e0: Running coprocessor pre-open hook at 1732387664298Writing region info on filesystem at 1732387664298Initializing all the Stores at 1732387664299 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387664300 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387664300Cleaning up temporary data from old regions at 1732387664309 (+9 ms)Running coprocessor post-open hooks at 1732387664315 (+6 ms)Region opened successfully at 1732387664316 (+1 ms) 2024-11-23T18:47:44,318 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., pid=6, masterSystemTime=1732387664291 2024-11-23T18:47:44,321 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,321 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,322 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPEN, openSeqNum=2, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:44,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503 because future has completed 2024-11-23T18:47:44,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T18:47:44,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503 in 190 msec 2024-11-23T18:47:44,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T18:47:44,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN in 357 msec 2024-11-23T18:47:44,335 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T18:47:44,336 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732387664336"}]},"ts":"1732387664336"} 2024-11-23T18:47:44,339 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-23T18:47:44,341 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T18:47:44,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 421 msec 2024-11-23T18:47:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T18:47:44,568 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-23T18:47:44,568 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-23T18:47:44,571 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:47:44,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-23T18:47:44,580 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T18:47:44,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-23T18:47:44,593 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=2] 2024-11-23T18:47:44,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e2564e420fa6e11419482705c96414e0, source=2eda60943ea1,41643,1732387649503, destination=2eda60943ea1,36631,1732387649346, warming up region on 2eda60943ea1,36631,1732387649346 2024-11-23T18:47:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:47:44,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e2564e420fa6e11419482705c96414e0, source=2eda60943ea1,41643,1732387649503, destination=2eda60943ea1,36631,1732387649346, running balancer 2024-11-23T18:47:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE 2024-11-23T18:47:44,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:47:44,614 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE 2024-11-23T18:47:44,616 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=CLOSING, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:44,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(7855): Warmup {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:44,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE because future has completed 2024-11-23T18:47:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:44,620 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T18:47:44,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503}] 2024-11-23T18:47:44,620 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,621 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf1 2024-11-23T18:47:44,621 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:44,622 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:44,622 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,623 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf2 2024-11-23T18:47:44,623 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:44,624 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:47:44,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:47:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387664624Disabling compacts and flushes for region at 1732387664624Disabling writes for close at 1732387664624Writing region close event to WAL at 1732387664625 (+1 ms)Closed at 1732387664625 2024-11-23T18:47:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T18:47:44,778 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,778 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T18:47:44,779 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:47:44,779 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,779 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,779 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:47:44,779 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,779 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing e2564e420fa6e11419482705c96414e0 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-23T18:47:44,798 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/1a869df772884b249288d38146b46e6f is 35, key is r1/cf1:q/1732387664596/Put/seqid=0 2024-11-23T18:47:44,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741887_1065 (size=4783) 2024-11-23T18:47:44,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741887_1065 (size=4783) 2024-11-23T18:47:44,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741887_1065 (size=4783) 2024-11-23T18:47:44,807 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/1a869df772884b249288d38146b46e6f 2024-11-23T18:47:44,815 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/1a869df772884b249288d38146b46e6f as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f 2024-11-23T18:47:44,822 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f, entries=1, sequenceid=5, filesize=4.7 K 2024-11-23T18:47:44,824 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e2564e420fa6e11419482705c96414e0 in 44ms, sequenceid=5, compaction requested=false 2024-11-23T18:47:44,824 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-23T18:47:44,829 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T18:47:44,831 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:44,831 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387664779Running coprocessor pre-close hooks at 1732387664779Disabling compacts and flushes for region at 1732387664779Disabling writes for close at 1732387664779Obtaining lock to block concurrent updates at 1732387664779Preparing flush snapshotting stores in e2564e420fa6e11419482705c96414e0 at 1732387664779Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732387664779Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. at 1732387664780 (+1 ms)Flushing e2564e420fa6e11419482705c96414e0/cf1: creating writer at 1732387664780Flushing e2564e420fa6e11419482705c96414e0/cf1: appending metadata at 1732387664797 (+17 ms)Flushing e2564e420fa6e11419482705c96414e0/cf1: closing flushed file at 1732387664798 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2556ef75: reopening flushed file at 1732387664814 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e2564e420fa6e11419482705c96414e0 in 44ms, sequenceid=5, compaction requested=false at 1732387664824 (+10 ms)Writing region close event to WAL at 1732387664825 (+1 ms)Running coprocessor post-close hooks at 1732387664829 (+4 ms)Closed at 1732387664831 (+2 ms) 2024-11-23T18:47:44,832 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e2564e420fa6e11419482705c96414e0 move to 2eda60943ea1,36631,1732387649346 record at close sequenceid=5 2024-11-23T18:47:44,835 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:44,836 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=CLOSED 2024-11-23T18:47:44,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503 because future has completed 2024-11-23T18:47:44,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T18:47:44,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503 in 220 msec 2024-11-23T18:47:44,845 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE; state=CLOSED, location=2eda60943ea1,36631,1732387649346; forceNewPlan=false, retain=false 2024-11-23T18:47:44,996 INFO [2eda60943ea1:37311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T18:47:44,997 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPENING, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:44,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE because future has completed 2024-11-23T18:47:44,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346}] 2024-11-23T18:47:45,161 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,162 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:45,162 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,163 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:45,163 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,163 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,167 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,168 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf1 2024-11-23T18:47:45,168 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:45,176 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f 2024-11-23T18:47:45,176 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:45,176 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,177 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf2 2024-11-23T18:47:45,177 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:45,178 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:45,178 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,179 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,180 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,181 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,181 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,182 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:47:45,184 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,185 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e2564e420fa6e11419482705c96414e0; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61240783, jitterRate=-0.08744122087955475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:47:45,185 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,186 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e2564e420fa6e11419482705c96414e0: Running coprocessor pre-open hook at 1732387665163Writing region info on filesystem at 1732387665163Initializing all the Stores at 1732387665166 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387665166Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387665166Cleaning up temporary data from old regions at 1732387665181 (+15 ms)Running coprocessor post-open hooks at 1732387665185 (+4 ms)Region opened successfully at 1732387665186 (+1 ms) 2024-11-23T18:47:45,188 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., pid=9, masterSystemTime=1732387665152 2024-11-23T18:47:45,190 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,191 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,191 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPEN, openSeqNum=9, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:45,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346 because future has completed 2024-11-23T18:47:45,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T18:47:45,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346 in 198 msec 2024-11-23T18:47:45,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE in 589 msec 2024-11-23T18:47:45,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:47:45,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:47:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:56398 deadline: 1732387725227, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=36631 startCode=1732387649346. As of locationSeqNum=5. 2024-11-23T18:47:45,236 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=36631 startCode=1732387649346. As of locationSeqNum=5. 2024-11-23T18:47:45,237 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=36631 startCode=1732387649346. As of locationSeqNum=5. 2024-11-23T18:47:45,237 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,36631,1732387649346, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=36631 startCode=1732387649346. As of locationSeqNum=5. 2024-11-23T18:47:45,345 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:47:45,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:47:45,359 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e2564e420fa6e11419482705c96414e0 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-23T18:47:45,381 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/476829323cdf4c9f884229264dfbb602 is 29, key is r1/cf1:/1732387665349/DeleteFamily/seqid=0 2024-11-23T18:47:45,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741888_1066 (size=4906) 2024-11-23T18:47:45,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741888_1066 (size=4906) 2024-11-23T18:47:45,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741888_1066 (size=4906) 2024-11-23T18:47:45,393 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,402 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,420 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf2/4892cb6d5a0145debe64b83cd1f8d0cd is 29, key is r1/cf2:/1732387665349/DeleteFamily/seqid=0 2024-11-23T18:47:45,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741889_1067 (size=4906) 2024-11-23T18:47:45,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741889_1067 (size=4906) 2024-11-23T18:47:45,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741889_1067 (size=4906) 2024-11-23T18:47:45,429 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf2/4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,435 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/476829323cdf4c9f884229264dfbb602 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,444 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,445 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602, entries=1, sequenceid=12, filesize=4.8 K 2024-11-23T18:47:45,446 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf2/4892cb6d5a0145debe64b83cd1f8d0cd as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,454 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,454 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd, entries=1, sequenceid=12, filesize=4.8 K 2024-11-23T18:47:45,456 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for e2564e420fa6e11419482705c96414e0 in 96ms, sequenceid=12, compaction requested=false 2024-11-23T18:47:45,456 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e2564e420fa6e11419482705c96414e0: 2024-11-23T18:47:45,458 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T18:47:45,459 DEBUG [Time-limited test {}] regionserver.HStore(1541): e2564e420fa6e11419482705c96414e0/cf1 is initiating major compaction (all files) 2024-11-23T18:47:45,460 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:47:45,460 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:45,460 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e2564e420fa6e11419482705c96414e0/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,461 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f, hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp, totalSize=9.5 K 2024-11-23T18:47:45,462 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1a869df772884b249288d38146b46e6f, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732387664596 2024-11-23T18:47:45,463 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 476829323cdf4c9f884229264dfbb602, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-23T18:47:45,479 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e2564e420fa6e11419482705c96414e0#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:47:45,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741890_1068 (size=4626) 2024-11-23T18:47:45,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741890_1068 (size=4626) 2024-11-23T18:47:45,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741890_1068 (size=4626) 2024-11-23T18:47:45,498 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf1/eb956d57cb384a1c87b0ec6d435c319b as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/eb956d57cb384a1c87b0ec6d435c319b 2024-11-23T18:47:45,517 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in e2564e420fa6e11419482705c96414e0/cf1 of e2564e420fa6e11419482705c96414e0 into eb956d57cb384a1c87b0ec6d435c319b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:47:45,517 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e2564e420fa6e11419482705c96414e0: 2024-11-23T18:47:45,517 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-23T18:47:45,517 DEBUG [Time-limited test {}] regionserver.HStore(1541): e2564e420fa6e11419482705c96414e0/cf2 is initiating major compaction (all files) 2024-11-23T18:47:45,517 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T18:47:45,517 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T18:47:45,518 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e2564e420fa6e11419482705c96414e0/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,518 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd] into tmpdir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp, totalSize=4.8 K 2024-11-23T18:47:45,519 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4892cb6d5a0145debe64b83cd1f8d0cd, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-23T18:47:45,525 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e2564e420fa6e11419482705c96414e0#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T18:47:45,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741891_1069 (size=4592) 2024-11-23T18:47:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741891_1069 (size=4592) 2024-11-23T18:47:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741891_1069 (size=4592) 2024-11-23T18:47:45,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/.tmp/cf2/1ec15a6d865e4f3896d44db889d0270a as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/1ec15a6d865e4f3896d44db889d0270a 2024-11-23T18:47:45,551 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in e2564e420fa6e11419482705c96414e0/cf2 of e2564e420fa6e11419482705c96414e0 into 1ec15a6d865e4f3896d44db889d0270a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:47:45,551 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e2564e420fa6e11419482705c96414e0: 2024-11-23T18:47:45,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e2564e420fa6e11419482705c96414e0, source=2eda60943ea1,36631,1732387649346, destination=2eda60943ea1,41643,1732387649503, warming up region on 2eda60943ea1,41643,1732387649503 2024-11-23T18:47:45,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e2564e420fa6e11419482705c96414e0, source=2eda60943ea1,36631,1732387649346, destination=2eda60943ea1,41643,1732387649503, running balancer 2024-11-23T18:47:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE 2024-11-23T18:47:45,559 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE 2024-11-23T18:47:45,560 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=CLOSING, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:45,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(7855): Warmup {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:45,561 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,563 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf1 2024-11-23T18:47:45,563 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:45,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE because future has completed 2024-11-23T18:47:45,564 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T18:47:45,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346}] 2024-11-23T18:47:45,572 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f 2024-11-23T18:47:45,578 INFO [StoreFileOpener-e2564e420fa6e11419482705c96414e0-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,578 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,584 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/eb956d57cb384a1c87b0ec6d435c319b 2024-11-23T18:47:45,584 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:45,585 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,586 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf2 2024-11-23T18:47:45,586 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:45,594 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/1ec15a6d865e4f3896d44db889d0270a 2024-11-23T18:47:45,601 INFO [StoreFileOpener-e2564e420fa6e11419482705c96414e0-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,601 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,601 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:47:45,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:47:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41643 {}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387665601Disabling compacts and flushes for region at 1732387665601Disabling writes for close at 1732387665602 (+1 ms)Writing region close event to WAL at 1732387665603 (+1 ms)Closed at 1732387665603 2024-11-23T18:47:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-23T18:47:45,718 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,718 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T18:47:45,718 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:47:45,718 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,719 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,719 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:47:45,719 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,719 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f, hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602] to archive 2024-11-23T18:47:45,722 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:47:45,726 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/1a869df772884b249288d38146b46e6f 2024-11-23T18:47:45,727 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602 to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/476829323cdf4c9f884229264dfbb602 2024-11-23T18:47:45,739 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd] to archive 2024-11-23T18:47:45,740 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:47:45,742 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/4892cb6d5a0145debe64b83cd1f8d0cd 2024-11-23T18:47:45,747 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-23T18:47:45,748 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:45,748 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387665718Running coprocessor pre-close hooks at 1732387665718Disabling compacts and flushes for region at 1732387665718Disabling writes for close at 1732387665719 (+1 ms)Writing region close event to WAL at 1732387665743 (+24 ms)Running coprocessor post-close hooks at 1732387665748 (+5 ms)Closed at 1732387665748 2024-11-23T18:47:45,748 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding e2564e420fa6e11419482705c96414e0 move to 2eda60943ea1,41643,1732387649503 record at close sequenceid=12 2024-11-23T18:47:45,750 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:45,751 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=CLOSED 2024-11-23T18:47:45,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346 because future has completed 2024-11-23T18:47:45,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-23T18:47:45,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346 in 191 msec 2024-11-23T18:47:45,759 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE; state=CLOSED, location=2eda60943ea1,41643,1732387649503; forceNewPlan=false, retain=false 2024-11-23T18:47:45,909 INFO [2eda60943ea1:37311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T18:47:45,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPENING, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:45,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE because future has completed 2024-11-23T18:47:45,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503}] 2024-11-23T18:47:46,074 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,074 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:46,075 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,075 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:46,075 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,075 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,076 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,077 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf1 2024-11-23T18:47:46,077 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:46,084 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/eb956d57cb384a1c87b0ec6d435c319b 2024-11-23T18:47:46,084 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:46,084 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,085 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf2 2024-11-23T18:47:46,085 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:46,091 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/1ec15a6d865e4f3896d44db889d0270a 2024-11-23T18:47:46,092 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:46,092 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,093 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,094 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,095 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,095 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,096 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:47:46,098 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,099 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened e2564e420fa6e11419482705c96414e0; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65294841, jitterRate=-0.027031049132347107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:47:46,099 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,099 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for e2564e420fa6e11419482705c96414e0: Running coprocessor pre-open hook at 1732387666075Writing region info on filesystem at 1732387666075Initializing all the Stores at 1732387666076 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387666076Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387666076Cleaning up temporary data from old regions at 1732387666095 (+19 ms)Running coprocessor post-open hooks at 1732387666099 (+4 ms)Region opened successfully at 1732387666099 2024-11-23T18:47:46,100 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., pid=12, masterSystemTime=1732387666070 2024-11-23T18:47:46,102 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,103 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,103 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPEN, openSeqNum=18, regionLocation=2eda60943ea1,41643,1732387649503 2024-11-23T18:47:46,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503 because future has completed 2024-11-23T18:47:46,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-23T18:47:46,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,41643,1732387649503 in 191 msec 2024-11-23T18:47:46,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, REOPEN/MOVE in 553 msec 2024-11-23T18:47:46,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:47:46,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:47:46,164 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2eda60943ea1,41643,1732387649503: testing ***** 2024-11-23T18:47:46,164 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-23T18:47:46,167 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-23T18:47:46,168 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-23T18:47:46,173 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-23T18:47:46,174 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-23T18:47:46,182 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 227763200 }, "NonHeapMemoryUsage": { "committed": 170262528, "init": 7667712, "max": -1, "used": 167528320 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2eda60943ea1", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2eda60943ea1", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63996", "tag.serverName": "2eda60943ea1,37545,1732387649456", "tag.clusterId": "36796ac2-6f7b-4ddf-9283-66f1c579705e", "tag.Context": "regionserver", "tag.Hostname": "2eda60943ea1", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732387649456, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-23T18:47:46,185 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37311 {}] master.MasterRpcServices(700): 2eda60943ea1,41643,1732387649503 reported a fatal error: ***** ABORTING region server 2eda60943ea1,41643,1732387649503: testing ***** 2024-11-23T18:47:46,187 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,41643,1732387649503' ***** 2024-11-23T18:47:46,187 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-23T18:47:46,187 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:47:46,188 INFO [RS:2;2eda60943ea1:41643 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-23T18:47:46,188 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:47:46,188 INFO [RS:2;2eda60943ea1:41643 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-23T18:47:46,188 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(3091): Received CLOSE for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:48232 deadline: 1732387726188, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=41643 startCode=1732387649503. As of locationSeqNum=12. 2024-11-23T18:47:46,188 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(956): aborting server 2eda60943ea1,41643,1732387649503 2024-11-23T18:47:46,189 INFO [RS:2;2eda60943ea1:41643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:47:46,189 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:47:46,189 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,36631,1732387649346, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,36631,1732387649346, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=41643 startCode=1732387649503. As of locationSeqNum=12. 2024-11-23T18:47:46,189 INFO [RS:2;2eda60943ea1:41643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2eda60943ea1:41643. 2024-11-23T18:47:46,189 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,189 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,36631,1732387649346, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=41643 startCode=1732387649503. As of locationSeqNum=12. 2024-11-23T18:47:46,189 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,189 DEBUG [RS:2;2eda60943ea1:41643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:47:46,189 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:47:46,189 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,36631,1732387649346, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2eda60943ea1 port=41643 startCode=1732387649503. As of locationSeqNum=12. 2024-11-23T18:47:46,189 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,189 DEBUG [RS:2;2eda60943ea1:41643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:47:46,190 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:47:46,190 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:47:46,190 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:47:46,190 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:47:46,190 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:47:46,190 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e2564e420fa6e11419482705c96414e0=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.} 2024-11-23T18:47:46,190 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:47:46,190 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:47:46,190 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:47:46,190 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:47:46,190 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:47:46,190 DEBUG [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,41643,1732387649503 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:46,191 ERROR [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-23T18:47:46,191 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,191 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:47:46,191 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387666189Running coprocessor pre-close hooks at 1732387666189Disabling compacts and flushes for region at 1732387666189Disabling writes for close at 1732387666189Writing region close event to WAL at 1732387666191 (+2 ms)Running coprocessor post-close hooks at 1732387666191Closed at 1732387666191 2024-11-23T18:47:46,191 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:47:46,191 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387666190Running coprocessor pre-close hooks at 1732387666190Disabling compacts and flushes for region at 1732387666190Disabling writes for close at 1732387666190Writing region close event to WAL at 1732387666191 (+1 ms)Running coprocessor post-close hooks at 1732387666191Closed at 1732387666191 2024-11-23T18:47:46,191 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:46,191 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:47:46,192 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,41643,1732387649503 aborting 2024-11-23T18:47:46,192 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,41643,1732387649503 aborting 2024-11-23T18:47:46,192 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 from cache 2024-11-23T18:47:46,243 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:47:46,294 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:46,297 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1] 2024-11-23T18:47:46,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,41643,1732387649503 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,41643,1732387649503 aborting 2024-11-23T18:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2eda60943ea1,41643,1732387649503 aborting 2024-11-23T18:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 from cache 2024-11-23T18:47:46,391 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,41643,1732387649503; all regions closed. 2024-11-23T18:47:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741836_1012 (size=6845) 2024-11-23T18:47:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741836_1012 (size=6845) 2024-11-23T18:47:46,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741836_1012 (size=6845) 2024-11-23T18:47:46,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741835_1011 (size=2142) 2024-11-23T18:47:46,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741835_1011 (size=2142) 2024-11-23T18:47:46,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741835_1011 (size=2142) 2024-11-23T18:47:46,401 DEBUG [RS:2;2eda60943ea1:41643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:47:46,401 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:47:46,401 INFO [RS:2;2eda60943ea1:41643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:47:46,401 INFO [RS:2;2eda60943ea1:41643 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T18:47:46,401 INFO [RS:2;2eda60943ea1:41643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:47:46,401 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:47:46,402 INFO [RS:2;2eda60943ea1:41643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41643 2024-11-23T18:47:46,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:47:46,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,41643,1732387649503 2024-11-23T18:47:46,483 INFO [RS:2;2eda60943ea1:41643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:47:46,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,41643,1732387649503] 2024-11-23T18:47:46,503 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,41643,1732387649503 already deleted, retry=false 2024-11-23T18:47:46,504 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2eda60943ea1,41643,1732387649503 on 2eda60943ea1,37311,1732387648561 2024-11-23T18:47:46,504 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:46,505 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1] 2024-11-23T18:47:46,507 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2eda60943ea1:41643 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2eda60943ea1/172.17.0.2:41643 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:46,507 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1, error=java.net.ConnectException: Call to address=2eda60943ea1:41643 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2eda60943ea1/172.17.0.2:41643 2024-11-23T18:47:46,507 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-23T18:47:46,507 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 from cache 2024-11-23T18:47:46,508 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 2eda60943ea1:41643 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2eda60943ea1/172.17.0.2:41643 2024-11-23T18:47:46,509 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2eda60943ea1,41643,1732387649503, splitWal=true, meta=true 2024-11-23T18:47:46,512 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 2eda60943ea1,41643,1732387649503 (carryingMeta=true) 2eda60943ea1,41643,1732387649503/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@799502b9[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T18:47:46,512 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2eda60943ea1,41643,1732387649503, splitWal=true, meta=true 2024-11-23T18:47:46,513 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2eda60943ea1,41643,1732387649503, splitWal=true, meta=true, isMeta: true 2024-11-23T18:47:46,515 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting 2024-11-23T18:47:46,516 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2eda60943ea1,41643,1732387649503 WAL count=1, meta=true 2024-11-23T18:47:46,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta}] 2024-11-23T18:47:46,523 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2eda60943ea1,37545,1732387649456 2024-11-23T18:47:46,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, worker=2eda60943ea1,37545,1732387649456}] 2024-11-23T18:47:46,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:47:46,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41643-0x1016905acb30003, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:47:46,594 INFO [RS:2;2eda60943ea1:41643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:47:46,594 INFO [RS:2;2eda60943ea1:41643 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,41643,1732387649503; zookeeper connection closed. 2024-11-23T18:47:46,595 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74c43e1c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74c43e1c 2024-11-23T18:47:46,691 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T18:47:46,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T18:47:46,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-23T18:47:46,714 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, size=6.7 K (6845bytes) 2024-11-23T18:47:46,714 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta 2024-11-23T18:47:46,715 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta after 1ms 2024-11-23T18:47:46,717 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta took 3ms 2024-11-23T18:47:46,723 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-23T18:47:46,724 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta so closing down 2024-11-23T18:47:46,724 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:46,726 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp 2024-11-23T18:47:46,727 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp 2024-11-23T18:47:46,727 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:46,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741892_1070 (size=6604) 2024-11-23T18:47:46,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741892_1070 (size=6604) 2024-11-23T18:47:46,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741892_1070 (size=6604) 2024-11-23T18:47:46,740 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:46,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:47:46,741 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 23 ms; skipped=1; WAL=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-23T18:47:46,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, journal: Splitting hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, size=6.7 K (6845bytes) at 1732387666714Finishing writing output for hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta so closing down at 1732387666724 (+10 ms)Creating recovered edits writer path=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp at 1732387666727 (+3 ms)3 split writer threads finished at 1732387666727Closed recovered edits writer path=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732387666740 (+13 ms)Rename recovered edits hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta.temp to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732387666741 (+1 ms)Processed 16 edits across 1 Regions in 23 ms; skipped=1; WAL=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1732387666741 2024-11-23T18:47:46,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta 2024-11-23T18:47:46,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-23T18:47:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-23T18:47:46,749 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs 2024-11-23T18:47:46,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-23T18:47:46,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, worker=2eda60943ea1,37545,1732387649456 in 224 msec 2024-11-23T18:47:46,753 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=2eda60943ea1,37545,1732387649456 2024-11-23T18:47:46,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T18:47:46,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2eda60943ea1%2C41643%2C1732387649503.meta.1732387651464.meta, worker=2eda60943ea1,37545,1732387649456 in 237 msec 2024-11-23T18:47:46,758 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting dir is empty, no logs to split. 2024-11-23T18:47:46,758 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2eda60943ea1,41643,1732387649503 WAL count=0, meta=true 2024-11-23T18:47:46,758 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2eda60943ea1,41643,1732387649503 WAL splitting is done? wals=0, meta=true 2024-11-23T18:47:46,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T18:47:46,762 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:47:46,763 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T18:47:46,814 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:46,815 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1] 2024-11-23T18:47:46,816 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:41643 this server is in the failed servers list 2024-11-23T18:47:46,816 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:41643 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:46,816 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:46,816 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 from cache 2024-11-23T18:47:46,913 DEBUG [2eda60943ea1:37311 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T18:47:46,913 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:47:46,914 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:47:46,914 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:47:46,914 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:47:46,914 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:47:46,914 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:47:46,914 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:47:46,914 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:47:46,915 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:46,917 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,36631,1732387649346, state=OPENING 2024-11-23T18:47:46,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:46,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:46,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:46,967 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T18:47:46,967 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:46,967 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:46,967 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:46,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,36631,1732387649346}] 2024-11-23T18:47:47,131 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T18:47:47,131 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:47,132 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T18:47:47,134 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2eda60943ea1%2C36631%2C1732387649346.meta, suffix=.meta, logDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346, archiveDir=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs, maxLogs=32 2024-11-23T18:47:47,149 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346/2eda60943ea1%2C36631%2C1732387649346.meta.1732387667135.meta, exclude list is [], retry=0 2024-11-23T18:47:47,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:47,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:47,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:47,154 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,36631,1732387649346/2eda60943ea1%2C36631%2C1732387649346.meta.1732387667135.meta 2024-11-23T18:47:47,154 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:47,154 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:47,155 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T18:47:47,155 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T18:47:47,155 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T18:47:47,155 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T18:47:47,155 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:47,155 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T18:47:47,155 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T18:47:47,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T18:47:47,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T18:47:47,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:47,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:47,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T18:47:47,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T18:47:47,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:47,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:47,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T18:47:47,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T18:47:47,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:47,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:47,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T18:47:47,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T18:47:47,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:47,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T18:47:47,162 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T18:47:47,163 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740 2024-11-23T18:47:47,164 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740 2024-11-23T18:47:47,165 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:47:47,169 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:47:47,169 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-23T18:47:47,193 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/info/526c76d86ee540d2b06222dcb0fdac33 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0./info:regioninfo/1732387666103/Put/seqid=0 2024-11-23T18:47:47,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741894_1072 (size=11177) 2024-11-23T18:47:47,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741894_1072 (size=11177) 2024-11-23T18:47:47,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741894_1072 (size=11177) 2024-11-23T18:47:47,199 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/info/526c76d86ee540d2b06222dcb0fdac33 2024-11-23T18:47:47,219 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/ns/8b2b8489aeb349ab9c337be242dd2585 is 43, key is default/ns:d/1732387651691/Put/seqid=0 2024-11-23T18:47:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741895_1073 (size=5153) 2024-11-23T18:47:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741895_1073 (size=5153) 2024-11-23T18:47:47,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741895_1073 (size=5153) 2024-11-23T18:47:47,225 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/ns/8b2b8489aeb349ab9c337be242dd2585 2024-11-23T18:47:47,245 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/table/95d20e628e3041cdb348c4060918f2f0 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732387664336/Put/seqid=0 2024-11-23T18:47:47,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741896_1074 (size=5431) 2024-11-23T18:47:47,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741896_1074 (size=5431) 2024-11-23T18:47:47,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741896_1074 (size=5431) 2024-11-23T18:47:47,256 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/table/95d20e628e3041cdb348c4060918f2f0 2024-11-23T18:47:47,262 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/info/526c76d86ee540d2b06222dcb0fdac33 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/info/526c76d86ee540d2b06222dcb0fdac33 2024-11-23T18:47:47,269 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/info/526c76d86ee540d2b06222dcb0fdac33, entries=36, sequenceid=18, filesize=10.9 K 2024-11-23T18:47:47,270 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/ns/8b2b8489aeb349ab9c337be242dd2585 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/ns/8b2b8489aeb349ab9c337be242dd2585 2024-11-23T18:47:47,277 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/ns/8b2b8489aeb349ab9c337be242dd2585, entries=2, sequenceid=18, filesize=5.0 K 2024-11-23T18:47:47,278 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/table/95d20e628e3041cdb348c4060918f2f0 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/table/95d20e628e3041cdb348c4060918f2f0 2024-11-23T18:47:47,284 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/table/95d20e628e3041cdb348c4060918f2f0, entries=2, sequenceid=18, filesize=5.3 K 2024-11-23T18:47:47,285 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 116ms, sequenceid=18, compaction requested=false; wal=null 2024-11-23T18:47:47,286 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T18:47:47,287 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T18:47:47,287 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T18:47:47,288 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T18:47:47,289 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T18:47:47,292 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T18:47:47,293 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60859884, jitterRate=-0.09311705827713013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T18:47:47,293 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T18:47:47,293 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732387667155Writing region info on filesystem at 1732387667155Initializing all the Stores at 1732387667156 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387667156Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387667156Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387667156Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732387667156Obtaining lock to block concurrent updates at 1732387667169 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1732387667169Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732387667170 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732387667170Flushing 1588230740/info: creating writer at 1732387667170Flushing 1588230740/info: appending metadata at 1732387667192 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732387667192Flushing 1588230740/ns: creating writer at 1732387667205 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732387667218 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732387667218Flushing 1588230740/table: creating writer at 1732387667230 (+12 ms)Flushing 1588230740/table: appending metadata at 1732387667244 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732387667245 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d8c3a0a: reopening flushed file at 1732387667261 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33dd9cc5: reopening flushed file at 1732387667269 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a79c58: reopening flushed file at 1732387667277 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 116ms, sequenceid=18, compaction requested=false; wal=null at 1732387667285 (+8 ms)Cleaning up temporary data from old regions at 1732387667287 (+2 ms)Running coprocessor post-open hooks at 1732387667293 (+6 ms)Region opened successfully at 1732387667293 2024-11-23T18:47:47,294 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732387667121 2024-11-23T18:47:47,296 DEBUG [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T18:47:47,297 INFO [RS_OPEN_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T18:47:47,297 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:47,298 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2eda60943ea1,36631,1732387649346, state=OPEN 2024-11-23T18:47:47,324 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:47,326 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,36631,1732387649346, seqNum=-1] 2024-11-23T18:47:47,343 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2eda60943ea1,36631,1732387649346, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-23T18:47:47,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:47,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:47,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T18:47:47,352 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:47,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:47,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:47,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T18:47:47,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-23T18:47:47,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2eda60943ea1,36631,1732387649346 in 385 msec 2024-11-23T18:47:47,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-23T18:47:47,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 599 msec 2024-11-23T18:47:47,363 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2eda60943ea1,41643,1732387649503 had 2 regions 2024-11-23T18:47:47,364 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2eda60943ea1,41643,1732387649503, splitWal=true, meta=true, isMeta: false 2024-11-23T18:47:47,366 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2eda60943ea1,41643,1732387649503 WAL count=1, meta=false 2024-11-23T18:47:47,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2eda60943ea1%2C41643%2C1732387649503.1732387651085}] 2024-11-23T18:47:47,368 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:47,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C41643%2C1732387649503.1732387651085, worker=2eda60943ea1,36631,1732387649346}] 2024-11-23T18:47:47,445 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:41643 this server is in the failed servers list 2024-11-23T18:47:47,445 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:41643 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:47,445 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:47,445 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=12 from cache 2024-11-23T18:47:47,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-23T18:47:47,536 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085, size=2.1 K (2142bytes) 2024-11-23T18:47:47,536 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 2024-11-23T18:47:47,536 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 after 0ms 2024-11-23T18:47:47,539 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 took 3ms 2024-11-23T18:47:47,541 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for e2564e420fa6e11419482705c96414e0: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-23T18:47:47,542 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 so closing down 2024-11-23T18:47:47,542 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:47,542 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:47,542 INFO [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-23T18:47:47,542 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085, journal: Splitting hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085, size=2.1 K (2142bytes) at 1732387667536Finishing writing output for hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 so closing down at 1732387667542 (+6 ms)3 split writer threads finished at 1732387667542Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732387667542 2024-11-23T18:47:47,542 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 2024-11-23T18:47:47,542 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2eda60943ea1:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-23T18:47:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37311 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-23T18:47:47,547 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting/2eda60943ea1%2C41643%2C1732387649503.1732387651085 to hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs 2024-11-23T18:47:47,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-23T18:47:47,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2eda60943ea1%2C41643%2C1732387649503.1732387651085, worker=2eda60943ea1,36631,1732387649346 in 178 msec 2024-11-23T18:47:47,551 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:47,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-23T18:47:47,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2eda60943ea1%2C41643%2C1732387649503.1732387651085, worker=2eda60943ea1,36631,1732387649346 in 185 msec 2024-11-23T18:47:47,555 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/WALs/2eda60943ea1,41643,1732387649503-splitting dir is empty, no logs to split. 2024-11-23T18:47:47,555 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2eda60943ea1,41643,1732387649503 WAL count=0, meta=false 2024-11-23T18:47:47,555 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2eda60943ea1,41643,1732387649503 WAL splitting is done? wals=0, meta=false 2024-11-23T18:47:47,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN}] 2024-11-23T18:47:47,559 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN 2024-11-23T18:47:47,560 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T18:47:47,660 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18] 2024-11-23T18:47:47,660 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:41643 this server is in the failed servers list 2024-11-23T18:47:47,661 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:41643 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:47,661 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:47,661 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18 from cache 2024-11-23T18:47:47,711 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(204): Hosts are {2eda60943ea1=0} racks are {/default-rack=0} 2024-11-23T18:47:47,711 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T18:47:47,711 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T18:47:47,711 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T18:47:47,711 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T18:47:47,711 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T18:47:47,711 INFO [2eda60943ea1:37311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T18:47:47,711 DEBUG [2eda60943ea1:37311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T18:47:47,712 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPENING, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:47,713 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2eda60943ea1:41643 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2eda60943ea1/172.17.0.2:41643 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:47,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1, error=java.net.ConnectException: Call to address=2eda60943ea1:41643 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2eda60943ea1/172.17.0.2:41643 2024-11-23T18:47:47,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-23T18:47:47,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,41643,1732387649503, seqNum=-1 from cache 2024-11-23T18:47:47,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 2eda60943ea1:41643 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2eda60943ea1/172.17.0.2:41643 2024-11-23T18:47:47,825 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T18:47:47,825 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2eda60943ea1,36631,1732387649346, seqNum=-1] 2024-11-23T18:47:47,825 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T18:47:47,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T18:47:47,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN because future has completed 2024-11-23T18:47:47,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346}] 2024-11-23T18:47:47,969 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18] 2024-11-23T18:47:47,970 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2eda60943ea1:41643 this server is in the failed servers list 2024-11-23T18:47:47,970 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2eda60943ea1:41643 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:47,970 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2eda60943ea1:41643 2024-11-23T18:47:47,970 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,41643,1732387649503, seqNum=18 from cache 2024-11-23T18:47:48,021 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:48,022 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => e2564e420fa6e11419482705c96414e0, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:48,022 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,022 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:48,023 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,023 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,025 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,026 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf1 2024-11-23T18:47:48,026 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,032 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf1/eb956d57cb384a1c87b0ec6d435c319b 2024-11-23T18:47:48,032 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,032 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,033 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2564e420fa6e11419482705c96414e0 columnFamilyName cf2 2024-11-23T18:47:48,033 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,039 DEBUG [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/cf2/1ec15a6d865e4f3896d44db889d0270a 2024-11-23T18:47:48,040 INFO [StoreOpener-e2564e420fa6e11419482705c96414e0-1 {}] regionserver.HStore(327): Store=e2564e420fa6e11419482705c96414e0/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,040 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,041 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,042 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,043 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,043 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,043 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T18:47:48,045 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,045 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened e2564e420fa6e11419482705c96414e0; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60786749, jitterRate=-0.09420685470104218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T18:47:48,045 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2564e420fa6e11419482705c96414e0 2024-11-23T18:47:48,046 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for e2564e420fa6e11419482705c96414e0: Running coprocessor pre-open hook at 1732387668023Writing region info on filesystem at 1732387668023Initializing all the Stores at 1732387668024 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668024Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668025 (+1 ms)Cleaning up temporary data from old regions at 1732387668043 (+18 ms)Running coprocessor post-open hooks at 1732387668045 (+2 ms)Region opened successfully at 1732387668046 (+1 ms) 2024-11-23T18:47:48,047 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., pid=21, masterSystemTime=1732387668014 2024-11-23T18:47:48,049 DEBUG [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:48,049 INFO [RS_OPEN_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:47:48,050 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e2564e420fa6e11419482705c96414e0, regionState=OPEN, openSeqNum=18, regionLocation=2eda60943ea1,36631,1732387649346 2024-11-23T18:47:48,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346 because future has completed 2024-11-23T18:47:48,057 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37311 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2eda60943ea1,36631,1732387649346, table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-23T18:47:48,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-23T18:47:48,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e2564e420fa6e11419482705c96414e0, server=2eda60943ea1,36631,1732387649346 in 229 msec 2024-11-23T18:47:48,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-23T18:47:48,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e2564e420fa6e11419482705c96414e0, ASSIGN in 503 msec 2024-11-23T18:47:48,062 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 2eda60943ea1,41643,1732387649503 after splitting done 2024-11-23T18:47:48,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 2eda60943ea1,41643,1732387649503, splitWal=true, meta=true in 1.5570 sec 2024-11-23T18:47:48,481 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0., hostname=2eda60943ea1,36631,1732387649346, seqNum=18] 2024-11-23T18:47:48,496 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 412), OpenFileDescriptor=1048 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=307 (was 334), ProcessCount=11 (was 11), AvailableMemoryMB=8747 (was 8862) 2024-11-23T18:47:48,497 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-23T18:47:48,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=8746 2024-11-23T18:47:48,510 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-23T18:47:48,529 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:48,530 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:48,531 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:48,534 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63143825, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-63143825, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:48,545 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63143825/hregion-63143825.1732387668534, exclude list is [], retry=0 2024-11-23T18:47:48,548 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:48,548 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:48,548 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:48,550 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63143825/hregion-63143825.1732387668534 2024-11-23T18:47:48,551 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:48,551 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fbc89e15abfda4ab3d934712ed480c45, NAME => 'testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:48,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741898_1076 (size=67) 2024-11-23T18:47:48,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741898_1076 (size=67) 2024-11-23T18:47:48,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741898_1076 (size=67) 2024-11-23T18:47:48,560 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:48,562 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,563 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName a 2024-11-23T18:47:48,563 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,563 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,564 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,565 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName b 2024-11-23T18:47:48,565 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,565 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,565 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,567 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName c 2024-11-23T18:47:48,567 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,567 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,569 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,569 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,570 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:48,571 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,573 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:48,573 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbc89e15abfda4ab3d934712ed480c45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60324203, jitterRate=-0.10109932720661163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:48,574 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbc89e15abfda4ab3d934712ed480c45: Writing region info on filesystem at 1732387668560Initializing all the Stores at 1732387668561 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668561Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668561Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668562 (+1 ms)Cleaning up temporary data from old regions at 1732387668569 (+7 ms)Region opened successfully at 1732387668574 (+5 ms) 2024-11-23T18:47:48,574 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fbc89e15abfda4ab3d934712ed480c45, disabling compactions & flushes 2024-11-23T18:47:48,574 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,574 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,574 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. after waiting 0 ms 2024-11-23T18:47:48,574 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,574 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,574 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fbc89e15abfda4ab3d934712ed480c45: Waiting for close lock at 1732387668574Disabling compacts and flushes for region at 1732387668574Disabling writes for close at 1732387668574Writing region close event to WAL at 1732387668574Closed at 1732387668574 2024-11-23T18:47:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741897_1075 (size=93) 2024-11-23T18:47:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741897_1075 (size=93) 2024-11-23T18:47:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741897_1075 (size=93) 2024-11-23T18:47:48,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:48,579 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63143825:(num 1732387668534) 2024-11-23T18:47:48,580 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:48,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:48,593 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581, exclude list is [], retry=0 2024-11-23T18:47:48,596 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:48,597 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:48,597 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:48,599 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 2024-11-23T18:47:48,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:48,599 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fbc89e15abfda4ab3d934712ed480c45, NAME => 'testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:48,600 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:48,600 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,600 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,601 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,602 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName a 2024-11-23T18:47:48,602 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,603 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,603 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,604 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName b 2024-11-23T18:47:48,604 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,605 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,605 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,606 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName c 2024-11-23T18:47:48,606 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,606 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,606 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,608 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,609 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,609 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,610 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:48,611 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,612 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbc89e15abfda4ab3d934712ed480c45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64580187, jitterRate=-0.03768022358417511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:48,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbc89e15abfda4ab3d934712ed480c45: Writing region info on filesystem at 1732387668600Initializing all the Stores at 1732387668601 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668601Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668601Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668601Cleaning up temporary data from old regions at 1732387668609 (+8 ms)Region opened successfully at 1732387668613 (+4 ms) 2024-11-23T18:47:48,637 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fbc89e15abfda4ab3d934712ed480c45 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-23T18:47:48,653 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/a/ab70e6cad9ed4318bfa055d51e4283b4 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732387668613/Put/seqid=0 2024-11-23T18:47:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741900_1078 (size=5958) 2024-11-23T18:47:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741900_1078 (size=5958) 2024-11-23T18:47:48,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741900_1078 (size=5958) 2024-11-23T18:47:48,660 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/a/ab70e6cad9ed4318bfa055d51e4283b4 2024-11-23T18:47:48,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/b/514e719b4932474d842f0bcfe2ac5444 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732387668622/Put/seqid=0 2024-11-23T18:47:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741901_1079 (size=5958) 2024-11-23T18:47:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741901_1079 (size=5958) 2024-11-23T18:47:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741901_1079 (size=5958) 2024-11-23T18:47:48,686 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/b/514e719b4932474d842f0bcfe2ac5444 2024-11-23T18:47:48,705 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/c/e27ca4a664224cd281ffdfa836f4891f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732387668630/Put/seqid=0 2024-11-23T18:47:48,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741902_1080 (size=5958) 2024-11-23T18:47:48,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741902_1080 (size=5958) 2024-11-23T18:47:48,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741902_1080 (size=5958) 2024-11-23T18:47:48,712 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/c/e27ca4a664224cd281ffdfa836f4891f 2024-11-23T18:47:48,717 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/a/ab70e6cad9ed4318bfa055d51e4283b4 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/a/ab70e6cad9ed4318bfa055d51e4283b4 2024-11-23T18:47:48,723 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/a/ab70e6cad9ed4318bfa055d51e4283b4, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T18:47:48,724 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/b/514e719b4932474d842f0bcfe2ac5444 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/b/514e719b4932474d842f0bcfe2ac5444 2024-11-23T18:47:48,730 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/b/514e719b4932474d842f0bcfe2ac5444, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T18:47:48,731 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/c/e27ca4a664224cd281ffdfa836f4891f as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/c/e27ca4a664224cd281ffdfa836f4891f 2024-11-23T18:47:48,736 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/c/e27ca4a664224cd281ffdfa836f4891f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T18:47:48,737 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for fbc89e15abfda4ab3d934712ed480c45 in 100ms, sequenceid=33, compaction requested=false 2024-11-23T18:47:48,737 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fbc89e15abfda4ab3d934712ed480c45: 2024-11-23T18:47:48,737 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fbc89e15abfda4ab3d934712ed480c45, disabling compactions & flushes 2024-11-23T18:47:48,737 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,737 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,737 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. after waiting 0 ms 2024-11-23T18:47:48,737 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,739 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. 2024-11-23T18:47:48,739 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fbc89e15abfda4ab3d934712ed480c45: Waiting for close lock at 1732387668737Disabling compacts and flushes for region at 1732387668737Disabling writes for close at 1732387668737Writing region close event to WAL at 1732387668739 (+2 ms)Closed at 1732387668739 2024-11-23T18:47:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741899_1077 (size=5937) 2024-11-23T18:47:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741899_1077 (size=5937) 2024-11-23T18:47:48,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741899_1077 (size=5937) 2024-11-23T18:47:48,746 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/b/514e719b4932474d842f0bcfe2ac5444 to hdfs://localhost:45809/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/b/514e719b4932474d842f0bcfe2ac5444 2024-11-23T18:47:48,762 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581, size=5.8 K (5937bytes) 2024-11-23T18:47:48,762 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 2024-11-23T18:47:48,762 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 after 0ms 2024-11-23T18:47:48,764 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 took 3ms 2024-11-23T18:47:48,767 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 so closing down 2024-11-23T18:47:48,767 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:48,768 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387668581.temp 2024-11-23T18:47:48,769 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000003-wal.1732387668581.temp 2024-11-23T18:47:48,770 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:48,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741903_1081 (size=5223) 2024-11-23T18:47:48,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741903_1081 (size=5223) 2024-11-23T18:47:48,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741903_1081 (size=5223) 2024-11-23T18:47:48,777 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000003-wal.1732387668581.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:48,779 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000003-wal.1732387668581.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000032 2024-11-23T18:47:48,779 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-23T18:47:48,779 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581, journal: Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581, size=5.8 K (5937bytes) at 1732387668762Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 so closing down at 1732387668767 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000003-wal.1732387668581.temp at 1732387668769 (+2 ms)3 split writer threads finished at 1732387668770 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000003-wal.1732387668581.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732387668777 (+7 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000003-wal.1732387668581.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000032 at 1732387668779 (+2 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732387668779 2024-11-23T18:47:48,781 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668581 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387668581 2024-11-23T18:47:48,782 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000032 2024-11-23T18:47:48,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:48,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:48,796 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668784, exclude list is [], retry=0 2024-11-23T18:47:48,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:48,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:48,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:48,801 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668784 2024-11-23T18:47:48,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:48,801 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fbc89e15abfda4ab3d934712ed480c45, NAME => 'testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:48,802 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:48,802 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,802 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,803 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,804 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName a 2024-11-23T18:47:48,804 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,813 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/a/ab70e6cad9ed4318bfa055d51e4283b4 2024-11-23T18:47:48,813 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,813 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,815 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName b 2024-11-23T18:47:48,815 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,816 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,816 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,817 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbc89e15abfda4ab3d934712ed480c45 columnFamilyName c 2024-11-23T18:47:48,817 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,823 DEBUG [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/c/e27ca4a664224cd281ffdfa836f4891f 2024-11-23T18:47:48,823 INFO [StoreOpener-fbc89e15abfda4ab3d934712ed480c45-1 {}] regionserver.HStore(327): Store=fbc89e15abfda4ab3d934712ed480c45/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,823 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,824 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,825 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,826 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000032 2024-11-23T18:47:48,829 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000032 2024-11-23T18:47:48,829 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fbc89e15abfda4ab3d934712ed480c45 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-23T18:47:48,844 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/b/13a24a8c215a48f79ac0e4cdfb15d663 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732387668622/Put/seqid=0 2024-11-23T18:47:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741905_1083 (size=5958) 2024-11-23T18:47:48,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741905_1083 (size=5958) 2024-11-23T18:47:48,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741905_1083 (size=5958) 2024-11-23T18:47:48,856 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/b/13a24a8c215a48f79ac0e4cdfb15d663 2024-11-23T18:47:48,864 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/.tmp/b/13a24a8c215a48f79ac0e4cdfb15d663 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/b/13a24a8c215a48f79ac0e4cdfb15d663 2024-11-23T18:47:48,871 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/b/13a24a8c215a48f79ac0e4cdfb15d663, entries=10, sequenceid=32, filesize=5.8 K 2024-11-23T18:47:48,871 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for fbc89e15abfda4ab3d934712ed480c45 in 42ms, sequenceid=32, compaction requested=false; wal=null 2024-11-23T18:47:48,872 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/0000000000000000032 2024-11-23T18:47:48,873 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,874 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,874 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:48,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fbc89e15abfda4ab3d934712ed480c45 2024-11-23T18:47:48,878 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/fbc89e15abfda4ab3d934712ed480c45/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-23T18:47:48,880 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fbc89e15abfda4ab3d934712ed480c45; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61988591, jitterRate=-0.07629801332950592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:48,881 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fbc89e15abfda4ab3d934712ed480c45: Writing region info on filesystem at 1732387668802Initializing all the Stores at 1732387668803 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668803Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668803Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668803Obtaining lock to block concurrent updates at 1732387668830 (+27 ms)Preparing flush snapshotting stores in fbc89e15abfda4ab3d934712ed480c45 at 1732387668830Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732387668830Flushing stores of testReplayEditsWrittenViaHRegion,,1732387668529.fbc89e15abfda4ab3d934712ed480c45. at 1732387668830Flushing fbc89e15abfda4ab3d934712ed480c45/b: creating writer at 1732387668830Flushing fbc89e15abfda4ab3d934712ed480c45/b: appending metadata at 1732387668843 (+13 ms)Flushing fbc89e15abfda4ab3d934712ed480c45/b: closing flushed file at 1732387668843Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f1ee2d7: reopening flushed file at 1732387668863 (+20 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for fbc89e15abfda4ab3d934712ed480c45 in 42ms, sequenceid=32, compaction requested=false; wal=null at 1732387668871 (+8 ms)Cleaning up temporary data from old regions at 1732387668874 (+3 ms)Region opened successfully at 1732387668880 (+6 ms) 2024-11-23T18:47:48,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-23T18:47:48,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:48,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-23T18:47:48,897 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:48,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-23T18:47:48,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:48,899 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-23T18:47:48,899 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:48,905 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:58598 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:51266 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:56084 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:51022 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:55934 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:58452 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=307 (was 307), ProcessCount=11 (was 11), AvailableMemoryMB=8730 (was 8746) 2024-11-23T18:47:48,905 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-23T18:47:48,918 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=8730 2024-11-23T18:47:48,918 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-23T18:47:48,940 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:48,943 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:48,944 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:48,947 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45120874, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-45120874, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:48,966 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45120874/hregion-45120874.1732387668947, exclude list is [], retry=0 2024-11-23T18:47:48,969 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:48,969 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:48,969 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:48,971 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45120874/hregion-45120874.1732387668947 2024-11-23T18:47:48,972 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:48,972 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 429f7ce77b475343ff29850c28574db8, NAME => 'testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741907_1085 (size=68) 2024-11-23T18:47:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741907_1085 (size=68) 2024-11-23T18:47:48,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741907_1085 (size=68) 2024-11-23T18:47:48,985 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:48,988 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,989 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName a 2024-11-23T18:47:48,989 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,990 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,990 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,991 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName b 2024-11-23T18:47:48,991 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,992 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,992 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,993 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName c 2024-11-23T18:47:48,993 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:48,994 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:48,994 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,995 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,995 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,996 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,996 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:48,997 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:48,998 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,000 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:49,001 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 429f7ce77b475343ff29850c28574db8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73905847, jitterRate=0.10128293931484222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:49,002 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 429f7ce77b475343ff29850c28574db8: Writing region info on filesystem at 1732387668985Initializing all the Stores at 1732387668986 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668986Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668987 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387668988 (+1 ms)Cleaning up temporary data from old regions at 1732387668996 (+8 ms)Region opened successfully at 1732387669002 (+6 ms) 2024-11-23T18:47:49,002 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 429f7ce77b475343ff29850c28574db8, disabling compactions & flushes 2024-11-23T18:47:49,002 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:49,002 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:49,002 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. after waiting 0 ms 2024-11-23T18:47:49,002 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:49,003 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:49,003 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 429f7ce77b475343ff29850c28574db8: Waiting for close lock at 1732387669002Disabling compacts and flushes for region at 1732387669002Disabling writes for close at 1732387669002Writing region close event to WAL at 1732387669003 (+1 ms)Closed at 1732387669003 2024-11-23T18:47:49,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741906_1084 (size=93) 2024-11-23T18:47:49,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741906_1084 (size=93) 2024-11-23T18:47:49,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741906_1084 (size=93) 2024-11-23T18:47:49,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:49,009 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45120874:(num 1732387668947) 2024-11-23T18:47:49,009 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:49,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:49,028 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012, exclude list is [], retry=0 2024-11-23T18:47:49,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:49,032 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:49,032 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:49,033 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 2024-11-23T18:47:49,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:49,090 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 429f7ce77b475343ff29850c28574db8, NAME => 'testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:49,092 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,092 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:49,092 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,092 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,094 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,095 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName a 2024-11-23T18:47:49,096 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:49,096 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:49,096 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,097 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName b 2024-11-23T18:47:49,097 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:49,098 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:49,098 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,098 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName c 2024-11-23T18:47:49,099 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:49,099 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:49,099 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,100 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,101 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,101 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,102 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,102 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:49,103 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,104 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 429f7ce77b475343ff29850c28574db8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66969031, jitterRate=-0.0020836740732192993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:49,104 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:49,104 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 429f7ce77b475343ff29850c28574db8: Running coprocessor pre-open hook at 1732387669093Writing region info on filesystem at 1732387669093Initializing all the Stores at 1732387669094 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387669094Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387669094Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387669094Cleaning up temporary data from old regions at 1732387669102 (+8 ms)Running coprocessor post-open hooks at 1732387669104 (+2 ms)Region opened successfully at 1732387669104 2024-11-23T18:47:49,116 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 429f7ce77b475343ff29850c28574db8 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-23T18:47:49,117 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:49,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:47:50,118 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:51,119 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:52,120 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:53,121 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:54,123 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:55,125 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:56,127 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:57,129 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:57,703 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T18:47:58,130 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 429f7ce77b475343ff29850c28574db8/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:47:58,133 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 429f7ce77b475343ff29850c28574db8: 2024-11-23T18:47:58,133 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:58,149 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 429f7ce77b475343ff29850c28574db8: 2024-11-23T18:47:58,149 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-23T18:47:58,149 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 429f7ce77b475343ff29850c28574db8, disabling compactions & flushes 2024-11-23T18:47:58,149 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:58,149 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:58,149 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. after waiting 0 ms 2024-11-23T18:47:58,149 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:58,149 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:58,150 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. 2024-11-23T18:47:58,150 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 429f7ce77b475343ff29850c28574db8: Waiting for close lock at 1732387678149Running coprocessor pre-close hooks at 1732387678149Disabling compacts and flushes for region at 1732387678149Disabling writes for close at 1732387678149Writing region close event to WAL at 1732387678149Running coprocessor post-close hooks at 1732387678149Closed at 1732387678149 2024-11-23T18:47:58,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741908_1086 (size=3941) 2024-11-23T18:47:58,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741908_1086 (size=3941) 2024-11-23T18:47:58,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741908_1086 (size=3941) 2024-11-23T18:47:58,167 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012, size=3.8 K (3941bytes) 2024-11-23T18:47:58,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 2024-11-23T18:47:58,168 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 after 1ms 2024-11-23T18:47:58,170 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 took 3ms 2024-11-23T18:47:58,172 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 so closing down 2024-11-23T18:47:58,172 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:58,173 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732387669012.temp 2024-11-23T18:47:58,174 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000004-wal.1732387669012.temp 2024-11-23T18:47:58,175 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741909_1087 (size=2983) 2024-11-23T18:47:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741909_1087 (size=2983) 2024-11-23T18:47:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741909_1087 (size=2983) 2024-11-23T18:47:58,181 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000004-wal.1732387669012.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:58,182 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000004-wal.1732387669012.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000026 2024-11-23T18:47:58,182 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-23T18:47:58,183 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012, journal: Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012, size=3.8 K (3941bytes) at 1732387678167Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 so closing down at 1732387678172 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000004-wal.1732387669012.temp at 1732387678174 (+2 ms)3 split writer threads finished at 1732387678175 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000004-wal.1732387669012.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732387678181 (+6 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000004-wal.1732387669012.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000026 at 1732387678182 (+1 ms)Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732387678182 2024-11-23T18:47:58,184 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387669012 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387669012 2024-11-23T18:47:58,185 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000026 2024-11-23T18:47:58,185 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:58,187 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:58,198 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387678187, exclude list is [], retry=0 2024-11-23T18:47:58,201 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:58,201 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:58,202 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:58,203 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387678187 2024-11-23T18:47:58,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:58,204 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 429f7ce77b475343ff29850c28574db8, NAME => 'testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:58,204 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,205 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,207 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,208 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName a 2024-11-23T18:47:58,208 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:58,208 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:58,208 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,209 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName b 2024-11-23T18:47:58,209 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:58,210 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:58,210 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,211 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 429f7ce77b475343ff29850c28574db8 columnFamilyName c 2024-11-23T18:47:58,211 DEBUG [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:58,212 INFO [StoreOpener-429f7ce77b475343ff29850c28574db8-1 {}] regionserver.HStore(327): Store=429f7ce77b475343ff29850c28574db8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:58,212 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,215 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000026 2024-11-23T18:47:58,252 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000026 2024-11-23T18:47:58,252 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 429f7ce77b475343ff29850c28574db8 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-23T18:47:58,266 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/a/d07e0307217640ffbecd94a5cad107d7 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732387678139/Put/seqid=0 2024-11-23T18:47:58,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741911_1089 (size=5523) 2024-11-23T18:47:58,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741911_1089 (size=5523) 2024-11-23T18:47:58,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741911_1089 (size=5523) 2024-11-23T18:47:58,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/a/d07e0307217640ffbecd94a5cad107d7 2024-11-23T18:47:58,696 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/b/d4f74f61bbec4327ab1478e63961d4be is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732387678134/Put/seqid=0 2024-11-23T18:47:58,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741912_1090 (size=5524) 2024-11-23T18:47:58,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741912_1090 (size=5524) 2024-11-23T18:47:58,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741912_1090 (size=5524) 2024-11-23T18:47:58,703 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/b/d4f74f61bbec4327ab1478e63961d4be 2024-11-23T18:47:58,724 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/c/a0b3c44654e44b4db05cf7ce49a3279a is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732387678137/Put/seqid=0 2024-11-23T18:47:58,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741913_1091 (size=5457) 2024-11-23T18:47:58,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741913_1091 (size=5457) 2024-11-23T18:47:58,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741913_1091 (size=5457) 2024-11-23T18:47:58,735 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/c/a0b3c44654e44b4db05cf7ce49a3279a 2024-11-23T18:47:58,742 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/a/d07e0307217640ffbecd94a5cad107d7 as hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/a/d07e0307217640ffbecd94a5cad107d7 2024-11-23T18:47:58,748 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/a/d07e0307217640ffbecd94a5cad107d7, entries=7, sequenceid=26, filesize=5.4 K 2024-11-23T18:47:58,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/b/d4f74f61bbec4327ab1478e63961d4be as hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/b/d4f74f61bbec4327ab1478e63961d4be 2024-11-23T18:47:58,755 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/b/d4f74f61bbec4327ab1478e63961d4be, entries=7, sequenceid=26, filesize=5.4 K 2024-11-23T18:47:58,757 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/.tmp/c/a0b3c44654e44b4db05cf7ce49a3279a as hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/c/a0b3c44654e44b4db05cf7ce49a3279a 2024-11-23T18:47:58,763 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/c/a0b3c44654e44b4db05cf7ce49a3279a, entries=6, sequenceid=26, filesize=5.3 K 2024-11-23T18:47:58,763 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 429f7ce77b475343ff29850c28574db8 in 511ms, sequenceid=26, compaction requested=false; wal=null 2024-11-23T18:47:58,764 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/0000000000000000026 2024-11-23T18:47:58,766 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,766 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,766 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:58,768 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,770 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsAfterAbortingFlush/429f7ce77b475343ff29850c28574db8/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-23T18:47:58,771 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 429f7ce77b475343ff29850c28574db8; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70093093, jitterRate=0.0444684773683548}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:58,771 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 429f7ce77b475343ff29850c28574db8 2024-11-23T18:47:58,771 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 429f7ce77b475343ff29850c28574db8: Running coprocessor pre-open hook at 1732387678205Writing region info on filesystem at 1732387678205Initializing all the Stores at 1732387678206 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387678206Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387678206Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387678206Obtaining lock to block concurrent updates at 1732387678252 (+46 ms)Preparing flush snapshotting stores in 429f7ce77b475343ff29850c28574db8 at 1732387678252Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732387678253 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732387668941.429f7ce77b475343ff29850c28574db8. at 1732387678253Flushing 429f7ce77b475343ff29850c28574db8/a: creating writer at 1732387678253Flushing 429f7ce77b475343ff29850c28574db8/a: appending metadata at 1732387678266 (+13 ms)Flushing 429f7ce77b475343ff29850c28574db8/a: closing flushed file at 1732387678266Flushing 429f7ce77b475343ff29850c28574db8/b: creating writer at 1732387678682 (+416 ms)Flushing 429f7ce77b475343ff29850c28574db8/b: appending metadata at 1732387678696 (+14 ms)Flushing 429f7ce77b475343ff29850c28574db8/b: closing flushed file at 1732387678696Flushing 429f7ce77b475343ff29850c28574db8/c: creating writer at 1732387678709 (+13 ms)Flushing 429f7ce77b475343ff29850c28574db8/c: appending metadata at 1732387678723 (+14 ms)Flushing 429f7ce77b475343ff29850c28574db8/c: closing flushed file at 1732387678723Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c4a6fd: reopening flushed file at 1732387678741 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7313e378: reopening flushed file at 1732387678748 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21761e32: reopening flushed file at 1732387678756 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 429f7ce77b475343ff29850c28574db8 in 511ms, sequenceid=26, compaction requested=false; wal=null at 1732387678763 (+7 ms)Cleaning up temporary data from old regions at 1732387678766 (+3 ms)Running coprocessor post-open hooks at 1732387678771 (+5 ms)Region opened successfully at 1732387678771 2024-11-23T18:47:58,793 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418 (was 415) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:33732 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:33756 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:40320 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:41652 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:41620 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:40312 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 307), ProcessCount=11 (was 11), AvailableMemoryMB=8715 (was 8730) 2024-11-23T18:47:58,793 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-23T18:47:58,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8715 2024-11-23T18:47:58,804 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-23T18:47:58,817 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:58,818 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:58,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:58,821 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16822248, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-16822248, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:58,834 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16822248/hregion-16822248.1732387678821, exclude list is [], retry=0 2024-11-23T18:47:58,837 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:58,837 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:58,838 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:58,840 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16822248/hregion-16822248.1732387678821 2024-11-23T18:47:58,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:58,840 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 794819ca94a5408517996e64f5da35b7, NAME => 'testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:58,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741915_1093 (size=61) 2024-11-23T18:47:58,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741915_1093 (size=61) 2024-11-23T18:47:58,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741915_1093 (size=61) 2024-11-23T18:47:58,851 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:58,852 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,853 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 794819ca94a5408517996e64f5da35b7 columnFamilyName a 2024-11-23T18:47:58,854 DEBUG [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:58,854 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(327): Store=794819ca94a5408517996e64f5da35b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:58,854 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,856 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,856 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,857 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,859 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:58,860 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 794819ca94a5408517996e64f5da35b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68387251, jitterRate=0.019049450755119324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:58,860 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 794819ca94a5408517996e64f5da35b7: Writing region info on filesystem at 1732387678851Initializing all the Stores at 1732387678852 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387678852Cleaning up temporary data from old regions at 1732387678856 (+4 ms)Region opened successfully at 1732387678860 (+4 ms) 2024-11-23T18:47:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 794819ca94a5408517996e64f5da35b7, disabling compactions & flushes 2024-11-23T18:47:58,861 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. after waiting 0 ms 2024-11-23T18:47:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,861 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 794819ca94a5408517996e64f5da35b7: Waiting for close lock at 1732387678860Disabling compacts and flushes for region at 1732387678860Disabling writes for close at 1732387678861 (+1 ms)Writing region close event to WAL at 1732387678861Closed at 1732387678861 2024-11-23T18:47:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741914_1092 (size=93) 2024-11-23T18:47:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741914_1092 (size=93) 2024-11-23T18:47:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741914_1092 (size=93) 2024-11-23T18:47:58,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:58,866 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16822248:(num 1732387678821) 2024-11-23T18:47:58,866 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:58,869 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:58,881 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869, exclude list is [], retry=0 2024-11-23T18:47:58,883 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:58,883 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:58,884 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:58,886 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 2024-11-23T18:47:58,886 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113)] 2024-11-23T18:47:58,886 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 794819ca94a5408517996e64f5da35b7, NAME => 'testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:58,886 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:58,886 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,886 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,888 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,889 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 794819ca94a5408517996e64f5da35b7 columnFamilyName a 2024-11-23T18:47:58,889 DEBUG [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:58,889 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(327): Store=794819ca94a5408517996e64f5da35b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:58,889 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,892 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,893 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:58,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-23T18:47:58,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-23T18:47:58,894 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 794819ca94a5408517996e64f5da35b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69871147, jitterRate=0.041161224246025085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:58,895 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 794819ca94a5408517996e64f5da35b7: Writing region info on filesystem at 1732387678886Initializing all the Stores at 1732387678887 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387678887Cleaning up temporary data from old regions at 1732387678892 (+5 ms)Region opened successfully at 1732387678895 (+3 ms) 2024-11-23T18:47:58,905 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 794819ca94a5408517996e64f5da35b7, disabling compactions & flushes 2024-11-23T18:47:58,906 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,906 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,906 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. after waiting 0 ms 2024-11-23T18:47:58,906 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,906 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,906 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. 2024-11-23T18:47:58,906 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 794819ca94a5408517996e64f5da35b7: Waiting for close lock at 1732387678905Disabling compacts and flushes for region at 1732387678905Disabling writes for close at 1732387678906 (+1 ms)Writing region close event to WAL at 1732387678906Closed at 1732387678906 2024-11-23T18:47:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741916_1094 (size=1623) 2024-11-23T18:47:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741916_1094 (size=1623) 2024-11-23T18:47:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741916_1094 (size=1623) 2024-11-23T18:47:58,925 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869, size=1.6 K (1623bytes) 2024-11-23T18:47:58,925 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 2024-11-23T18:47:58,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 after 1ms 2024-11-23T18:47:58,928 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 took 3ms 2024-11-23T18:47:58,930 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 so closing down 2024-11-23T18:47:58,930 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:47:58,932 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387678869.temp 2024-11-23T18:47:58,933 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000003-wal.1732387678869.temp 2024-11-23T18:47:58,933 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:47:58,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741917_1095 (size=1623) 2024-11-23T18:47:58,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741917_1095 (size=1623) 2024-11-23T18:47:58,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741917_1095 (size=1623) 2024-11-23T18:47:58,940 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000003-wal.1732387678869.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-23T18:47:58,942 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000003-wal.1732387678869.temp to hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000012 2024-11-23T18:47:58,942 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-23T18:47:58,942 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869, journal: Splitting hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869, size=1.6 K (1623bytes) at 1732387678925Finishing writing output for hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 so closing down at 1732387678930 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000003-wal.1732387678869.temp at 1732387678933 (+3 ms)3 split writer threads finished at 1732387678933Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000003-wal.1732387678869.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732387678940 (+7 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000003-wal.1732387678869.temp to hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000012 at 1732387678942 (+2 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732387678942 2024-11-23T18:47:58,944 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387678869 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387678869 2024-11-23T18:47:58,945 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000012 2024-11-23T18:47:59,307 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:59,309 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:59,322 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387679309, exclude list is [], retry=0 2024-11-23T18:47:59,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:59,325 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:59,326 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:59,327 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387679309 2024-11-23T18:47:59,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:59,328 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 794819ca94a5408517996e64f5da35b7, NAME => 'testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:59,328 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:59,328 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,328 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,331 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,332 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 794819ca94a5408517996e64f5da35b7 columnFamilyName a 2024-11-23T18:47:59,332 DEBUG [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,333 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(327): Store=794819ca94a5408517996e64f5da35b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,333 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,334 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,335 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,336 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000012 2024-11-23T18:47:59,338 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000012 2024-11-23T18:47:59,338 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 794819ca94a5408517996e64f5da35b7 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-23T18:47:59,356 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/.tmp/a/e6d30222518e491784770eeca820363f is 79, key is testDatalossWhenInputError/a:x0/1732387678895/Put/seqid=0 2024-11-23T18:47:59,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741919_1097 (size=5808) 2024-11-23T18:47:59,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741919_1097 (size=5808) 2024-11-23T18:47:59,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741919_1097 (size=5808) 2024-11-23T18:47:59,363 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/.tmp/a/e6d30222518e491784770eeca820363f 2024-11-23T18:47:59,373 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/.tmp/a/e6d30222518e491784770eeca820363f as hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/a/e6d30222518e491784770eeca820363f 2024-11-23T18:47:59,385 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/a/e6d30222518e491784770eeca820363f, entries=10, sequenceid=12, filesize=5.7 K 2024-11-23T18:47:59,385 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 794819ca94a5408517996e64f5da35b7 in 47ms, sequenceid=12, compaction requested=false; wal=null 2024-11-23T18:47:59,386 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/0000000000000000012 2024-11-23T18:47:59,387 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,387 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,390 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,393 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-23T18:47:59,394 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 794819ca94a5408517996e64f5da35b7; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66036951, jitterRate=-0.015972748398780823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:59,395 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 794819ca94a5408517996e64f5da35b7: Writing region info on filesystem at 1732387679328Initializing all the Stores at 1732387679330 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679330Obtaining lock to block concurrent updates at 1732387679338 (+8 ms)Preparing flush snapshotting stores in 794819ca94a5408517996e64f5da35b7 at 1732387679338Finished memstore snapshotting testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732387679338Flushing stores of testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7. at 1732387679338Flushing 794819ca94a5408517996e64f5da35b7/a: creating writer at 1732387679338Flushing 794819ca94a5408517996e64f5da35b7/a: appending metadata at 1732387679356 (+18 ms)Flushing 794819ca94a5408517996e64f5da35b7/a: closing flushed file at 1732387679356Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7952022: reopening flushed file at 1732387679371 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 794819ca94a5408517996e64f5da35b7 in 47ms, sequenceid=12, compaction requested=false; wal=null at 1732387679385 (+14 ms)Cleaning up temporary data from old regions at 1732387679387 (+2 ms)Region opened successfully at 1732387679395 (+8 ms) 2024-11-23T18:47:59,398 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 794819ca94a5408517996e64f5da35b7, NAME => 'testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:59,398 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732387678817.794819ca94a5408517996e64f5da35b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:59,398 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,399 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,400 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,401 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 794819ca94a5408517996e64f5da35b7 columnFamilyName a 2024-11-23T18:47:59,401 DEBUG [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,406 DEBUG [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/a/e6d30222518e491784770eeca820363f 2024-11-23T18:47:59,407 INFO [StoreOpener-794819ca94a5408517996e64f5da35b7-1 {}] regionserver.HStore(327): Store=794819ca94a5408517996e64f5da35b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,407 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,410 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,410 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,412 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 794819ca94a5408517996e64f5da35b7 2024-11-23T18:47:59,414 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testDatalossWhenInputError/794819ca94a5408517996e64f5da35b7/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-23T18:47:59,415 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 794819ca94a5408517996e64f5da35b7; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71282261, jitterRate=0.06218846142292023}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T18:47:59,415 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 794819ca94a5408517996e64f5da35b7: Writing region info on filesystem at 1732387679399Initializing all the Stores at 1732387679400 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679400Cleaning up temporary data from old regions at 1732387679410 (+10 ms)Region opened successfully at 1732387679415 (+5 ms) 2024-11-23T18:47:59,433 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:33810 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-657037303-172.17.0.2-1732387644132:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:40392 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:41652 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:40312 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:41704 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8718 (was 8715) - AvailableMemoryMB LEAK? - 2024-11-23T18:47:59,433 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-23T18:47:59,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8717 2024-11-23T18:47:59,446 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-23T18:47:59,462 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:59,464 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:47:59,465 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:47:59,467 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82125580, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-82125580, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:59,485 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82125580/hregion-82125580.1732387679468, exclude list is [], retry=0 2024-11-23T18:47:59,488 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:59,488 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:59,489 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:59,493 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82125580/hregion-82125580.1732387679468 2024-11-23T18:47:59,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:47:59,493 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 645b69fb03b130ea58c94c6522c54921, NAME => 'testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:47:59,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741921_1099 (size=63) 2024-11-23T18:47:59,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741921_1099 (size=63) 2024-11-23T18:47:59,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741921_1099 (size=63) 2024-11-23T18:47:59,508 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:59,509 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,511 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName a 2024-11-23T18:47:59,511 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,511 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,512 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,513 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName b 2024-11-23T18:47:59,513 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,514 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,514 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,516 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName c 2024-11-23T18:47:59,516 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,516 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,516 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,518 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,518 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,519 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:59,520 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,522 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:47:59,522 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 645b69fb03b130ea58c94c6522c54921; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60517036, jitterRate=-0.09822589159011841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:59,523 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 645b69fb03b130ea58c94c6522c54921: Writing region info on filesystem at 1732387679508Initializing all the Stores at 1732387679509 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679509Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679509Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679509Cleaning up temporary data from old regions at 1732387679518 (+9 ms)Region opened successfully at 1732387679523 (+5 ms) 2024-11-23T18:47:59,523 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 645b69fb03b130ea58c94c6522c54921, disabling compactions & flushes 2024-11-23T18:47:59,523 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:47:59,523 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:47:59,523 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. after waiting 0 ms 2024-11-23T18:47:59,523 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:47:59,524 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:47:59,524 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 645b69fb03b130ea58c94c6522c54921: Waiting for close lock at 1732387679523Disabling compacts and flushes for region at 1732387679523Disabling writes for close at 1732387679523Writing region close event to WAL at 1732387679524 (+1 ms)Closed at 1732387679524 2024-11-23T18:47:59,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741920_1098 (size=93) 2024-11-23T18:47:59,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741920_1098 (size=93) 2024-11-23T18:47:59,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741920_1098 (size=93) 2024-11-23T18:47:59,529 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:47:59,529 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82125580:(num 1732387679468) 2024-11-23T18:47:59,529 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:47:59,531 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:47:59,543 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531, exclude list is [], retry=0 2024-11-23T18:47:59,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:47:59,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:47:59,547 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:47:59,548 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 2024-11-23T18:47:59,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753),(127.0.0.1/127.0.0.1:37381:37381)] 2024-11-23T18:47:59,549 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 645b69fb03b130ea58c94c6522c54921, NAME => 'testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:47:59,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:47:59,549 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,549 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,553 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,554 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName a 2024-11-23T18:47:59,554 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,554 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,555 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,556 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName b 2024-11-23T18:47:59,556 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,556 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,556 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,557 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName c 2024-11-23T18:47:59,557 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:47:59,558 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:47:59,558 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,560 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,561 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,561 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,561 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:47:59,563 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:47:59,563 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 645b69fb03b130ea58c94c6522c54921; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72351004, jitterRate=0.07811397314071655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:47:59,564 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 645b69fb03b130ea58c94c6522c54921: Writing region info on filesystem at 1732387679549Initializing all the Stores at 1732387679550 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679550Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679553 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387679553Cleaning up temporary data from old regions at 1732387679561 (+8 ms)Region opened successfully at 1732387679564 (+3 ms) 2024-11-23T18:47:59,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732387679567/Put/seqid=0 2024-11-23T18:47:59,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741923_1101 (size=4875) 2024-11-23T18:47:59,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741923_1101 (size=4875) 2024-11-23T18:47:59,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741923_1101 (size=4875) 2024-11-23T18:47:59,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732387679581/Put/seqid=0 2024-11-23T18:47:59,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741924_1102 (size=4875) 2024-11-23T18:47:59,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741924_1102 (size=4875) 2024-11-23T18:47:59,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741924_1102 (size=4875) 2024-11-23T18:47:59,995 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732387679995/Put/seqid=0 2024-11-23T18:48:00,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741925_1103 (size=4875) 2024-11-23T18:48:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741925_1103 (size=4875) 2024-11-23T18:48:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741925_1103 (size=4875) 2024-11-23T18:48:00,007 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 645b69fb03b130ea58c94c6522c54921/a 2024-11-23T18:48:00,011 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-23T18:48:00,011 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:48:00,011 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 645b69fb03b130ea58c94c6522c54921/a 2024-11-23T18:48:00,016 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-23T18:48:00,016 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:48:00,016 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 645b69fb03b130ea58c94c6522c54921/a 2024-11-23T18:48:00,019 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-23T18:48:00,019 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T18:48:00,019 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 645b69fb03b130ea58c94c6522c54921 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-23T18:48:00,035 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/.tmp/a/e9d6843ddc1e4aa1a7aa624374c5f1cd is 55, key is testCompactedBulkLoadedFiles/a:a/1732387679564/Put/seqid=0 2024-11-23T18:48:00,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741926_1104 (size=5107) 2024-11-23T18:48:00,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741926_1104 (size=5107) 2024-11-23T18:48:00,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741926_1104 (size=5107) 2024-11-23T18:48:00,042 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/.tmp/a/e9d6843ddc1e4aa1a7aa624374c5f1cd 2024-11-23T18:48:00,047 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/.tmp/a/e9d6843ddc1e4aa1a7aa624374c5f1cd as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd 2024-11-23T18:48:00,052 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd, entries=1, sequenceid=4, filesize=5.0 K 2024-11-23T18:48:00,053 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 645b69fb03b130ea58c94c6522c54921 in 34ms, sequenceid=4, compaction requested=false 2024-11-23T18:48:00,053 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 645b69fb03b130ea58c94c6522c54921: 2024-11-23T18:48:00,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ 2024-11-23T18:48:00,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ 2024-11-23T18:48:00,057 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ 2024-11-23T18:48:00,057 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile0 into 645b69fb03b130ea58c94c6522c54921/a as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ - updating store file list. 2024-11-23T18:48:00,062 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 224df1a486954ae2882316cb12bd76ef_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:48:00,062 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ into 645b69fb03b130ea58c94c6522c54921/a 2024-11-23T18:48:00,062 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile0 into 645b69fb03b130ea58c94c6522c54921/a (new location: hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_) 2024-11-23T18:48:00,063 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile1 into 645b69fb03b130ea58c94c6522c54921/a as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ - updating store file list. 2024-11-23T18:48:00,067 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:48:00,068 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ into 645b69fb03b130ea58c94c6522c54921/a 2024-11-23T18:48:00,068 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile1 into 645b69fb03b130ea58c94c6522c54921/a (new location: hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_) 2024-11-23T18:48:00,069 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile2 into 645b69fb03b130ea58c94c6522c54921/a as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ - updating store file list. 2024-11-23T18:48:00,073 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4536847b34174b61a7905b7744ea6a15_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:48:00,073 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ into 645b69fb03b130ea58c94c6522c54921/a 2024-11-23T18:48:00,073 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45809/hbase/testCompactedBulkLoadedFiles/hfile2 into 645b69fb03b130ea58c94c6522c54921/a (new location: hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_) 2024-11-23T18:48:00,081 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T18:48:00,081 DEBUG [Time-limited test {}] regionserver.HStore(1541): 645b69fb03b130ea58c94c6522c54921/a is initiating major compaction (all files) 2024-11-23T18:48:00,081 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 645b69fb03b130ea58c94c6522c54921/a in testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:48:00,081 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_] into tmpdir=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/.tmp, totalSize=19.3 K 2024-11-23T18:48:00,082 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e9d6843ddc1e4aa1a7aa624374c5f1cd, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732387679564 2024-11-23T18:48:00,082 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 224df1a486954ae2882316cb12bd76ef_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T18:48:00,083 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T18:48:00,083 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4536847b34174b61a7905b7744ea6a15_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T18:48:00,095 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/.tmp/a/41370aaed17e4b37904bafbbccac051a is 55, key is testCompactedBulkLoadedFiles/a:a/1732387679564/Put/seqid=0 2024-11-23T18:48:00,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741927_1105 (size=6154) 2024-11-23T18:48:00,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741927_1105 (size=6154) 2024-11-23T18:48:00,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741927_1105 (size=6154) 2024-11-23T18:48:00,107 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/.tmp/a/41370aaed17e4b37904bafbbccac051a as hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/41370aaed17e4b37904bafbbccac051a 2024-11-23T18:48:00,114 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 645b69fb03b130ea58c94c6522c54921/a of 645b69fb03b130ea58c94c6522c54921 into 41370aaed17e4b37904bafbbccac051a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T18:48:00,115 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 645b69fb03b130ea58c94c6522c54921: 2024-11-23T18:48:00,115 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-23T18:48:00,115 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-23T18:48:00,145 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531, size=0 (0bytes) 2024-11-23T18:48:00,145 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 might be still open, length is 0 2024-11-23T18:48:00,145 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 2024-11-23T18:48:00,145 WARN [IPC Server handler 3 on default port 45809 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-23T18:48:00,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 after 1ms 2024-11-23T18:48:00,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:41724 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41724 dst: /127.0.0.1:34245 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34245 remote=/127.0.0.1:41724]. Total timeout mills is 60000, 59447 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:00,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:40424 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39151:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40424 dst: /127.0.0.1:39151 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:00,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:33850 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38589:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33850 dst: /127.0.0.1:38589 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:00,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741922_1106 (size=1659) 2024-11-23T18:48:00,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741922_1106 (size=1659) 2024-11-23T18:48:03,679 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:48:04,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 after 4002ms 2024-11-23T18:48:04,153 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 took 4007ms 2024-11-23T18:48:04,156 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531; continuing. 2024-11-23T18:48:04,156 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 so closing down 2024-11-23T18:48:04,156 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:48:04,158 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387679531.temp 2024-11-23T18:48:04,159 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000003-wal.1732387679531.temp 2024-11-23T18:48:04,159 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:48:04,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741928_1107 (size=698) 2024-11-23T18:48:04,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741928_1107 (size=698) 2024-11-23T18:48:04,166 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000003-wal.1732387679531.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T18:48:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741928_1107 (size=698) 2024-11-23T18:48:04,168 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000003-wal.1732387679531.temp to hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000008 2024-11-23T18:48:04,168 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:48:04,168 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531, journal: Splitting hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531, size=0 (0bytes) at 1732387680145Finishing writing output for hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 so closing down at 1732387684156 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000003-wal.1732387679531.temp at 1732387684159 (+3 ms)3 split writer threads finished at 1732387684159Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000003-wal.1732387679531.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732387684166 (+7 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000003-wal.1732387679531.temp to hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000008 at 1732387684168 (+2 ms)Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531, size=0, length=0, corrupted=false, cancelled=false at 1732387684168 2024-11-23T18:48:04,170 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387679531 2024-11-23T18:48:04,171 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000008 2024-11-23T18:48:04,171 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:48:04,173 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:48:04,188 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387684173, exclude list is [], retry=0 2024-11-23T18:48:04,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:48:04,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:48:04,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:48:04,193 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387684173 2024-11-23T18:48:04,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:48:04,193 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 645b69fb03b130ea58c94c6522c54921, NAME => 'testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:48:04,193 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:48:04,193 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,193 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,195 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,196 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName a 2024-11-23T18:48:04,196 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,202 DEBUG [StoreFileOpener-645b69fb03b130ea58c94c6522c54921-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 224df1a486954ae2882316cb12bd76ef_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:48:04,202 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ 2024-11-23T18:48:04,207 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/41370aaed17e4b37904bafbbccac051a 2024-11-23T18:48:04,211 DEBUG [StoreFileOpener-645b69fb03b130ea58c94c6522c54921-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4536847b34174b61a7905b7744ea6a15_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:48:04,211 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ 2024-11-23T18:48:04,216 DEBUG [StoreFileOpener-645b69fb03b130ea58c94c6522c54921-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T18:48:04,216 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ 2024-11-23T18:48:04,220 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd 2024-11-23T18:48:04,220 WARN [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@45118266 2024-11-23T18:48:04,220 WARN [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@45118266 2024-11-23T18:48:04,220 WARN [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@45118266 2024-11-23T18:48:04,220 WARN [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@45118266 2024-11-23T18:48:04,220 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd] to archive 2024-11-23T18:48:04,221 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T18:48:04,222 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ to hdfs://localhost:45809/hbase/archive/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/224df1a486954ae2882316cb12bd76ef_SeqId_4_ 2024-11-23T18:48:04,224 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ to hdfs://localhost:45809/hbase/archive/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/4536847b34174b61a7905b7744ea6a15_SeqId_4_ 2024-11-23T18:48:04,225 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ to hdfs://localhost:45809/hbase/archive/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_ 2024-11-23T18:48:04,226 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd to hdfs://localhost:45809/hbase/archive/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/a/e9d6843ddc1e4aa1a7aa624374c5f1cd 2024-11-23T18:48:04,226 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,226 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,227 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName b 2024-11-23T18:48:04,227 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,227 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,227 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,228 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 645b69fb03b130ea58c94c6522c54921 columnFamilyName c 2024-11-23T18:48:04,228 DEBUG [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,229 INFO [StoreOpener-645b69fb03b130ea58c94c6522c54921-1 {}] regionserver.HStore(327): Store=645b69fb03b130ea58c94c6522c54921/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,229 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,232 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000008 2024-11-23T18:48:04,235 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 645b69fb03b130ea58c94c6522c54921 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "645b69fb03b130ea58c94c6522c54921" family_name: "a" compaction_input: "e9d6843ddc1e4aa1a7aa624374c5f1cd" compaction_input: "224df1a486954ae2882316cb12bd76ef_SeqId_4_" compaction_input: "d7693c9cef7b46b29732cd9c65a002b4_SeqId_4_" compaction_input: "4536847b34174b61a7905b7744ea6a15_SeqId_4_" compaction_output: "41370aaed17e4b37904bafbbccac051a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-23T18:48:04,235 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-23T18:48:04,235 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000008 2024-11-23T18:48:04,236 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/0000000000000000008 2024-11-23T18:48:04,237 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,237 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,238 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:48:04,239 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 645b69fb03b130ea58c94c6522c54921 2024-11-23T18:48:04,241 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testCompactedBulkLoadedFiles/645b69fb03b130ea58c94c6522c54921/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T18:48:04,242 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 645b69fb03b130ea58c94c6522c54921; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70117563, jitterRate=0.04483310878276825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:48:04,242 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 645b69fb03b130ea58c94c6522c54921: Writing region info on filesystem at 1732387684194Initializing all the Stores at 1732387684194Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684194Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684195 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684195Cleaning up temporary data from old regions at 1732387684237 (+42 ms)Region opened successfully at 1732387684242 (+5 ms) 2024-11-23T18:48:04,245 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 645b69fb03b130ea58c94c6522c54921, disabling compactions & flushes 2024-11-23T18:48:04,245 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:48:04,245 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:48:04,245 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. after waiting 0 ms 2024-11-23T18:48:04,245 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:48:04,246 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732387679463.645b69fb03b130ea58c94c6522c54921. 2024-11-23T18:48:04,246 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 645b69fb03b130ea58c94c6522c54921: Waiting for close lock at 1732387684245Disabling compacts and flushes for region at 1732387684245Disabling writes for close at 1732387684245Writing region close event to WAL at 1732387684246 (+1 ms)Closed at 1732387684246 2024-11-23T18:48:04,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741929_1108 (size=93) 2024-11-23T18:48:04,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741929_1108 (size=93) 2024-11-23T18:48:04,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741929_1108 (size=93) 2024-11-23T18:48:04,251 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:48:04,251 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387684173) 2024-11-23T18:48:04,264 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=440 (was 428) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2047170302_22 at /127.0.0.1:55320 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:45809 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2047170302_22 at /127.0.0.1:48174 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:45809 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2047170302_22 at /127.0.0.1:54936 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8710 (was 8717) 2024-11-23T18:48:04,265 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-23T18:48:04,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=8706 2024-11-23T18:48:04,277 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-23T18:48:04,292 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:48:04,294 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T18:48:04,294 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T18:48:04,297 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91320457, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/hregion-91320457, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:48:04,310 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91320457/hregion-91320457.1732387684297, exclude list is [], retry=0 2024-11-23T18:48:04,312 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:48:04,313 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:48:04,313 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:48:04,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91320457/hregion-91320457.1732387684297 2024-11-23T18:48:04,315 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:48:04,315 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1f378bbc5e31213d1d98ef246284fdfc, NAME => 'testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45809/hbase 2024-11-23T18:48:04,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741931_1110 (size=67) 2024-11-23T18:48:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741931_1110 (size=67) 2024-11-23T18:48:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741931_1110 (size=67) 2024-11-23T18:48:04,327 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:48:04,328 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,329 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName a 2024-11-23T18:48:04,329 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,330 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,330 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,332 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName b 2024-11-23T18:48:04,332 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,332 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,332 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,333 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName c 2024-11-23T18:48:04,333 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,334 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,334 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,335 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,335 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,336 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,336 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,337 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:48:04,338 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,340 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T18:48:04,340 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f378bbc5e31213d1d98ef246284fdfc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66956270, jitterRate=-0.0022738277912139893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:48:04,340 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f378bbc5e31213d1d98ef246284fdfc: Writing region info on filesystem at 1732387684327Initializing all the Stores at 1732387684328 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684328Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684328Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684328Cleaning up temporary data from old regions at 1732387684336 (+8 ms)Region opened successfully at 1732387684340 (+4 ms) 2024-11-23T18:48:04,341 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f378bbc5e31213d1d98ef246284fdfc, disabling compactions & flushes 2024-11-23T18:48:04,341 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,341 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,341 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. after waiting 0 ms 2024-11-23T18:48:04,341 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,341 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,341 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f378bbc5e31213d1d98ef246284fdfc: Waiting for close lock at 1732387684341Disabling compacts and flushes for region at 1732387684341Disabling writes for close at 1732387684341Writing region close event to WAL at 1732387684341Closed at 1732387684341 2024-11-23T18:48:04,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741930_1109 (size=93) 2024-11-23T18:48:04,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741930_1109 (size=93) 2024-11-23T18:48:04,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741930_1109 (size=93) 2024-11-23T18:48:04,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:48:04,347 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91320457:(num 1732387684297) 2024-11-23T18:48:04,347 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:48:04,348 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:48:04,366 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349, exclude list is [], retry=0 2024-11-23T18:48:04,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:48:04,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:48:04,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:48:04,372 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 2024-11-23T18:48:04,372 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:48:04,372 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f378bbc5e31213d1d98ef246284fdfc, NAME => 'testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:48:04,372 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:48:04,372 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,372 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,374 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,375 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName a 2024-11-23T18:48:04,375 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,375 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,376 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,376 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName b 2024-11-23T18:48:04,376 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,377 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,377 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,378 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName c 2024-11-23T18:48:04,378 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,378 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,379 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,380 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,381 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,381 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,382 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:48:04,383 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,384 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f378bbc5e31213d1d98ef246284fdfc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63192079, jitterRate=-0.05836464464664459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:48:04,384 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f378bbc5e31213d1d98ef246284fdfc: Writing region info on filesystem at 1732387684373Initializing all the Stores at 1732387684374 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684374Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684374Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684374Cleaning up temporary data from old regions at 1732387684381 (+7 ms)Region opened successfully at 1732387684384 (+3 ms) 2024-11-23T18:48:04,392 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f378bbc5e31213d1d98ef246284fdfc 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-23T18:48:04,408 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/a/4eb7abc8fd464e62901854ded4bc090b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732387684384/Put/seqid=0 2024-11-23T18:48:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741933_1112 (size=5958) 2024-11-23T18:48:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741933_1112 (size=5958) 2024-11-23T18:48:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741933_1112 (size=5958) 2024-11-23T18:48:04,416 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/a/4eb7abc8fd464e62901854ded4bc090b 2024-11-23T18:48:04,422 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/a/4eb7abc8fd464e62901854ded4bc090b as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/a/4eb7abc8fd464e62901854ded4bc090b 2024-11-23T18:48:04,427 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/a/4eb7abc8fd464e62901854ded4bc090b, entries=10, sequenceid=13, filesize=5.8 K 2024-11-23T18:48:04,428 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 1f378bbc5e31213d1d98ef246284fdfc in 36ms, sequenceid=13, compaction requested=false 2024-11-23T18:48:04,428 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f378bbc5e31213d1d98ef246284fdfc: 2024-11-23T18:48:04,449 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f378bbc5e31213d1d98ef246284fdfc, disabling compactions & flushes 2024-11-23T18:48:04,449 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,449 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,449 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. after waiting 0 ms 2024-11-23T18:48:04,449 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,450 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,450 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:04,450 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f378bbc5e31213d1d98ef246284fdfc: Waiting for close lock at 1732387684449Disabling compacts and flushes for region at 1732387684449Disabling writes for close at 1732387684449Writing region close event to WAL at 1732387684450 (+1 ms)Closed at 1732387684450 2024-11-23T18:48:04,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741932_1111 (size=5869) 2024-11-23T18:48:04,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741932_1111 (size=5869) 2024-11-23T18:48:04,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741932_1111 (size=5869) 2024-11-23T18:48:04,475 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349, size=5.7 K (5869bytes) 2024-11-23T18:48:04,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 2024-11-23T18:48:04,476 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 after 1ms 2024-11-23T18:48:04,478 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 took 3ms 2024-11-23T18:48:04,481 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 so closing down 2024-11-23T18:48:04,481 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:48:04,482 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732387684349.temp 2024-11-23T18:48:04,483 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000003-wal.1732387684349.temp 2024-11-23T18:48:04,483 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:48:04,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741934_1113 (size=5223) 2024-11-23T18:48:04,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741934_1113 (size=5223) 2024-11-23T18:48:04,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741934_1113 (size=5223) 2024-11-23T18:48:04,493 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000003-wal.1732387684349.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T18:48:04,494 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000003-wal.1732387684349.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000035 2024-11-23T18:48:04,494 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-23T18:48:04,494 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349, journal: Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349, size=5.7 K (5869bytes) at 1732387684475Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 so closing down at 1732387684481 (+6 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000003-wal.1732387684349.temp at 1732387684483 (+2 ms)3 split writer threads finished at 1732387684483Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000003-wal.1732387684349.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732387684493 (+10 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000003-wal.1732387684349.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000035 at 1732387684494 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732387684494 2024-11-23T18:48:04,496 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684349 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387684349 2024-11-23T18:48:04,497 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000035 2024-11-23T18:48:04,497 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:48:04,499 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:48:04,516 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500, exclude list is [], retry=0 2024-11-23T18:48:04,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:48:04,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:48:04,519 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:48:04,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 2024-11-23T18:48:04,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:48:04,520 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f378bbc5e31213d1d98ef246284fdfc, NAME => 'testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.', STARTKEY => '', ENDKEY => ''} 2024-11-23T18:48:04,520 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:48:04,521 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,521 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,522 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,523 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName a 2024-11-23T18:48:04,523 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,529 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/a/4eb7abc8fd464e62901854ded4bc090b 2024-11-23T18:48:04,529 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,529 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,530 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName b 2024-11-23T18:48:04,530 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,530 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,531 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,531 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName c 2024-11-23T18:48:04,531 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:04,532 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:04,532 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,535 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,535 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000035 2024-11-23T18:48:04,539 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000035 2024-11-23T18:48:04,539 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f378bbc5e31213d1d98ef246284fdfc 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-23T18:48:04,559 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/b/29f18ec21e944cc4b2de2cb074604893 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732387684428/Put/seqid=0 2024-11-23T18:48:04,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741936_1115 (size=5958) 2024-11-23T18:48:04,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741936_1115 (size=5958) 2024-11-23T18:48:04,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741936_1115 (size=5958) 2024-11-23T18:48:04,566 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/b/29f18ec21e944cc4b2de2cb074604893 2024-11-23T18:48:04,585 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/c/ea5e667a35e34cea877b7390b1470db1 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732387684438/Put/seqid=0 2024-11-23T18:48:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741937_1116 (size=5958) 2024-11-23T18:48:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741937_1116 (size=5958) 2024-11-23T18:48:04,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741937_1116 (size=5958) 2024-11-23T18:48:04,594 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/c/ea5e667a35e34cea877b7390b1470db1 2024-11-23T18:48:04,600 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/b/29f18ec21e944cc4b2de2cb074604893 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/b/29f18ec21e944cc4b2de2cb074604893 2024-11-23T18:48:04,605 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/b/29f18ec21e944cc4b2de2cb074604893, entries=10, sequenceid=35, filesize=5.8 K 2024-11-23T18:48:04,606 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/c/ea5e667a35e34cea877b7390b1470db1 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/c/ea5e667a35e34cea877b7390b1470db1 2024-11-23T18:48:04,611 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/c/ea5e667a35e34cea877b7390b1470db1, entries=10, sequenceid=35, filesize=5.8 K 2024-11-23T18:48:04,612 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 1f378bbc5e31213d1d98ef246284fdfc in 72ms, sequenceid=35, compaction requested=false; wal=null 2024-11-23T18:48:04,613 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000035 2024-11-23T18:48:04,614 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,614 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,615 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:48:04,616 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:04,619 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-23T18:48:04,620 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f378bbc5e31213d1d98ef246284fdfc; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64994407, jitterRate=-0.031507864594459534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:48:04,620 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f378bbc5e31213d1d98ef246284fdfc: Writing region info on filesystem at 1732387684521Initializing all the Stores at 1732387684522 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684522Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684522Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387684522Obtaining lock to block concurrent updates at 1732387684539 (+17 ms)Preparing flush snapshotting stores in 1f378bbc5e31213d1d98ef246284fdfc at 1732387684539Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732387684540 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. at 1732387684540Flushing 1f378bbc5e31213d1d98ef246284fdfc/b: creating writer at 1732387684540Flushing 1f378bbc5e31213d1d98ef246284fdfc/b: appending metadata at 1732387684559 (+19 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/b: closing flushed file at 1732387684559Flushing 1f378bbc5e31213d1d98ef246284fdfc/c: creating writer at 1732387684571 (+12 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/c: appending metadata at 1732387684584 (+13 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/c: closing flushed file at 1732387684584Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@239f9d64: reopening flushed file at 1732387684599 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24c96d09: reopening flushed file at 1732387684605 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 1f378bbc5e31213d1d98ef246284fdfc in 72ms, sequenceid=35, compaction requested=false; wal=null at 1732387684612 (+7 ms)Cleaning up temporary data from old regions at 1732387684614 (+2 ms)Region opened successfully at 1732387684620 (+6 ms) 2024-11-23T18:48:04,700 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500, size=0 (0bytes) 2024-11-23T18:48:04,700 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 might be still open, length is 0 2024-11-23T18:48:04,701 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 2024-11-23T18:48:04,701 WARN [IPC Server handler 4 on default port 45809 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-23T18:48:04,701 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 after 0ms 2024-11-23T18:48:06,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:54984 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39151:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54984 dst: /127.0.0.1:39151 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39151 remote=/127.0.0.1:54984]. Total timeout mills is 60000, 57987 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:06,671 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:55380 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38589:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55380 dst: /127.0.0.1:38589 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:06,672 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922606282_22 at /127.0.0.1:48258 [Receiving block BP-657037303-172.17.0.2-1732387644132:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34245:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48258 dst: /127.0.0.1:34245 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:06,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741935_1117 (size=5215) 2024-11-23T18:48:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741935_1117 (size=5215) 2024-11-23T18:48:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741935_1117 (size=5215) 2024-11-23T18:48:08,702 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 after 4001ms 2024-11-23T18:48:08,738 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 took 4038ms 2024-11-23T18:48:08,740 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500; continuing. 2024-11-23T18:48:08,740 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 so closing down 2024-11-23T18:48:08,740 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T18:48:08,741 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732387684500.temp 2024-11-23T18:48:08,742 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000037-wal.1732387684500.temp 2024-11-23T18:48:08,743 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T18:48:08,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741938_1118 (size=5223) 2024-11-23T18:48:08,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741938_1118 (size=5223) 2024-11-23T18:48:08,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741938_1118 (size=5223) 2024-11-23T18:48:08,750 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000037-wal.1732387684500.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T18:48:08,751 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000037-wal.1732387684500.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000066 2024-11-23T18:48:08,751 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T18:48:08,751 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500, journal: Splitting hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500, size=0 (0bytes) at 1732387684700Finishing writing output for hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 so closing down at 1732387688740 (+4040 ms)Creating recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000037-wal.1732387684500.temp at 1732387688742 (+2 ms)3 split writer threads finished at 1732387688743 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000037-wal.1732387684500.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732387688750 (+7 ms)Rename recovered edits hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000037-wal.1732387684500.temp to hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000066 at 1732387688751 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500, size=0, length=0, corrupted=false, cancelled=false at 1732387688751 2024-11-23T18:48:08,753 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 to hdfs://localhost:45809/hbase/oldWALs/wal.1732387684500 2024-11-23T18:48:08,754 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000066 2024-11-23T18:48:08,754 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T18:48:08,755 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45809/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291, archiveDir=hdfs://localhost:45809/hbase/oldWALs, maxLogs=32 2024-11-23T18:48:08,767 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387688756, exclude list is [], retry=0 2024-11-23T18:48:08,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39151,DS-93877182-6065-4aee-9cc8-d3467a2d0e87,DISK] 2024-11-23T18:48:08,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34245,DS-be3c2a2f-a8ca-4816-b6b9-328bbe31f58c,DISK] 2024-11-23T18:48:08,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38589,DS-495cf48c-cc16-484c-9f2c-99cf787132a4,DISK] 2024-11-23T18:48:08,771 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387688756 2024-11-23T18:48:08,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37381:37381),(127.0.0.1/127.0.0.1:45113:45113),(127.0.0.1/127.0.0.1:36753:36753)] 2024-11-23T18:48:08,772 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T18:48:08,773 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,774 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName a 2024-11-23T18:48:08,774 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:08,780 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/a/4eb7abc8fd464e62901854ded4bc090b 2024-11-23T18:48:08,780 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:08,780 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,781 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName b 2024-11-23T18:48:08,781 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:08,786 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/b/29f18ec21e944cc4b2de2cb074604893 2024-11-23T18:48:08,786 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:08,786 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,787 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f378bbc5e31213d1d98ef246284fdfc columnFamilyName c 2024-11-23T18:48:08,787 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T18:48:08,792 DEBUG [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/c/ea5e667a35e34cea877b7390b1470db1 2024-11-23T18:48:08,792 INFO [StoreOpener-1f378bbc5e31213d1d98ef246284fdfc-1 {}] regionserver.HStore(327): Store=1f378bbc5e31213d1d98ef246284fdfc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T18:48:08,793 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,795 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,795 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000066 2024-11-23T18:48:08,800 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000066 2024-11-23T18:48:08,800 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f378bbc5e31213d1d98ef246284fdfc 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-23T18:48:08,815 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/a/c9e06b4d3da0476f92d5a3afcbf78179 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732387684628/Put/seqid=0 2024-11-23T18:48:08,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741940_1120 (size=5958) 2024-11-23T18:48:08,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741940_1120 (size=5958) 2024-11-23T18:48:08,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741940_1120 (size=5958) 2024-11-23T18:48:08,822 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/a/c9e06b4d3da0476f92d5a3afcbf78179 2024-11-23T18:48:08,841 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/b/c1da9ee3b1184bc19430b70eda881633 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732387684638/Put/seqid=0 2024-11-23T18:48:08,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741941_1121 (size=5958) 2024-11-23T18:48:08,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741941_1121 (size=5958) 2024-11-23T18:48:08,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741941_1121 (size=5958) 2024-11-23T18:48:08,848 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/b/c1da9ee3b1184bc19430b70eda881633 2024-11-23T18:48:08,867 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/c/9a8caacd850b48d8a44f4c4ce5bb4019 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732387684647/Put/seqid=0 2024-11-23T18:48:08,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741942_1122 (size=5958) 2024-11-23T18:48:08,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741942_1122 (size=5958) 2024-11-23T18:48:08,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741942_1122 (size=5958) 2024-11-23T18:48:08,874 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/c/9a8caacd850b48d8a44f4c4ce5bb4019 2024-11-23T18:48:08,880 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/a/c9e06b4d3da0476f92d5a3afcbf78179 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/a/c9e06b4d3da0476f92d5a3afcbf78179 2024-11-23T18:48:08,885 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/a/c9e06b4d3da0476f92d5a3afcbf78179, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T18:48:08,887 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/b/c1da9ee3b1184bc19430b70eda881633 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/b/c1da9ee3b1184bc19430b70eda881633 2024-11-23T18:48:08,892 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/b/c1da9ee3b1184bc19430b70eda881633, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T18:48:08,893 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/.tmp/c/9a8caacd850b48d8a44f4c4ce5bb4019 as hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/c/9a8caacd850b48d8a44f4c4ce5bb4019 2024-11-23T18:48:08,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-23T18:48:08,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-23T18:48:08,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-23T18:48:08,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-23T18:48:08,899 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/c/9a8caacd850b48d8a44f4c4ce5bb4019, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T18:48:08,899 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 1f378bbc5e31213d1d98ef246284fdfc in 99ms, sequenceid=66, compaction requested=false; wal=null 2024-11-23T18:48:08,900 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/0000000000000000066 2024-11-23T18:48:08,901 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,901 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,902 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T18:48:08,903 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f378bbc5e31213d1d98ef246284fdfc 2024-11-23T18:48:08,905 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/hbase/data/default/testReplayEditsWrittenViaHRegion/1f378bbc5e31213d1d98ef246284fdfc/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-23T18:48:08,906 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f378bbc5e31213d1d98ef246284fdfc; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70814566, jitterRate=0.05521926283836365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T18:48:08,906 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f378bbc5e31213d1d98ef246284fdfc: Writing region info on filesystem at 1732387688772Initializing all the Stores at 1732387688773 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387688773Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387688773Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732387688773Obtaining lock to block concurrent updates at 1732387688800 (+27 ms)Preparing flush snapshotting stores in 1f378bbc5e31213d1d98ef246284fdfc at 1732387688800Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732387688800Flushing stores of testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. at 1732387688800Flushing 1f378bbc5e31213d1d98ef246284fdfc/a: creating writer at 1732387688800Flushing 1f378bbc5e31213d1d98ef246284fdfc/a: appending metadata at 1732387688814 (+14 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/a: closing flushed file at 1732387688814Flushing 1f378bbc5e31213d1d98ef246284fdfc/b: creating writer at 1732387688827 (+13 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/b: appending metadata at 1732387688841 (+14 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/b: closing flushed file at 1732387688841Flushing 1f378bbc5e31213d1d98ef246284fdfc/c: creating writer at 1732387688852 (+11 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/c: appending metadata at 1732387688867 (+15 ms)Flushing 1f378bbc5e31213d1d98ef246284fdfc/c: closing flushed file at 1732387688867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dd78776: reopening flushed file at 1732387688879 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f1de5: reopening flushed file at 1732387688886 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78160131: reopening flushed file at 1732387688892 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 1f378bbc5e31213d1d98ef246284fdfc in 99ms, sequenceid=66, compaction requested=false; wal=null at 1732387688899 (+7 ms)Cleaning up temporary data from old regions at 1732387688901 (+2 ms)Region opened successfully at 1732387688906 (+5 ms) 2024-11-23T18:48:08,917 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f378bbc5e31213d1d98ef246284fdfc, disabling compactions & flushes 2024-11-23T18:48:08,917 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:08,917 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:08,917 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. after waiting 0 ms 2024-11-23T18:48:08,917 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:08,918 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732387684292.1f378bbc5e31213d1d98ef246284fdfc. 2024-11-23T18:48:08,919 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f378bbc5e31213d1d98ef246284fdfc: Waiting for close lock at 1732387688917Disabling compacts and flushes for region at 1732387688917Disabling writes for close at 1732387688917Writing region close event to WAL at 1732387688918 (+1 ms)Closed at 1732387688918 2024-11-23T18:48:08,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741939_1119 (size=93) 2024-11-23T18:48:08,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741939_1119 (size=93) 2024-11-23T18:48:08,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741939_1119 (size=93) 2024-11-23T18:48:08,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T18:48:08,923 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732387688756) 2024-11-23T18:48:08,936 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 440) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:45809 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1775252730_22 at /127.0.0.1:48298 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:45809 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1775252730_22 at /127.0.0.1:55034 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1775252730_22 at /127.0.0.1:55416 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=8690 (was 8706) 2024-11-23T18:48:08,936 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-23T18:48:08,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T18:48:08,936 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:48:08,937 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:48:08,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:08,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:08,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T18:48:08,937 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T18:48:08,937 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=639412996, stopped=false 2024-11-23T18:48:08,938 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2eda60943ea1,37311,1732387648561 2024-11-23T18:48:09,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:48:09,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:48:09,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T18:48:09,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:48:09,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:48:09,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:48:09,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:48:09,020 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T18:48:09,020 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:48:09,020 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:48:09,020 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:48:09,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:09,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T18:48:09,021 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,36631,1732387649346' ***** 2024-11-23T18:48:09,021 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:48:09,021 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2eda60943ea1,37545,1732387649456' ***** 2024-11-23T18:48:09,022 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T18:48:09,022 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:48:09,022 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T18:48:09,022 INFO [RS:0;2eda60943ea1:36631 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:48:09,022 INFO [RS:1;2eda60943ea1:37545 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T18:48:09,022 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:48:09,022 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T18:48:09,022 INFO [RS:1;2eda60943ea1:37545 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:48:09,022 INFO [RS:0;2eda60943ea1:36631 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T18:48:09,022 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(3091): Received CLOSE for e2564e420fa6e11419482705c96414e0 2024-11-23T18:48:09,023 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,37545,1732387649456 2024-11-23T18:48:09,023 INFO [RS:1;2eda60943ea1:37545 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(959): stopping server 2eda60943ea1,36631,1732387649346 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:48:09,023 INFO [RS:1;2eda60943ea1:37545 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2eda60943ea1:37545. 2024-11-23T18:48:09,023 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2564e420fa6e11419482705c96414e0, disabling compactions & flushes 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2eda60943ea1:36631. 2024-11-23T18:48:09,023 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:48:09,023 DEBUG [RS:1;2eda60943ea1:37545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:48:09,023 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:48:09,023 DEBUG [RS:0;2eda60943ea1:36631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T18:48:09,023 DEBUG [RS:1;2eda60943ea1:37545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:09,023 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. after waiting 0 ms 2024-11-23T18:48:09,023 DEBUG [RS:0;2eda60943ea1:36631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:09,023 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:48:09,023 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,37545,1732387649456; all regions closed. 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:48:09,023 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T18:48:09,024 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T18:48:09,024 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e2564e420fa6e11419482705c96414e0=testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0.} 2024-11-23T18:48:09,024 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T18:48:09,024 DEBUG [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e2564e420fa6e11419482705c96414e0 2024-11-23T18:48:09,024 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T18:48:09,024 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T18:48:09,024 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T18:48:09,024 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T18:48:09,025 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-23T18:48:09,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741834_1010 (size=93) 2024-11-23T18:48:09,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741834_1010 (size=93) 2024-11-23T18:48:09,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741834_1010 (size=93) 2024-11-23T18:48:09,029 DEBUG [RS:1;2eda60943ea1:37545 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs 2024-11-23T18:48:09,029 INFO [RS:1;2eda60943ea1:37545 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C37545%2C1732387649456:(num 1732387651085) 2024-11-23T18:48:09,029 DEBUG [RS:1;2eda60943ea1:37545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:09,029 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:48:09,029 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e2564e420fa6e11419482705c96414e0/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-23T18:48:09,029 INFO [RS:1;2eda60943ea1:37545 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:48:09,029 INFO [RS:1;2eda60943ea1:37545 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T18:48:09,030 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T18:48:09,030 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:48:09,030 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T18:48:09,030 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T18:48:09,030 INFO [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:48:09,030 INFO [RS:1;2eda60943ea1:37545 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:48:09,030 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2564e420fa6e11419482705c96414e0: Waiting for close lock at 1732387689023Running coprocessor pre-close hooks at 1732387689023Disabling compacts and flushes for region at 1732387689023Disabling writes for close at 1732387689023Writing region close event to WAL at 1732387689024 (+1 ms)Running coprocessor post-close hooks at 1732387689030 (+6 ms)Closed at 1732387689030 2024-11-23T18:48:09,030 INFO [RS:1;2eda60943ea1:37545 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37545 2024-11-23T18:48:09,030 DEBUG [RS_CLOSE_REGION-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0. 2024-11-23T18:48:09,031 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/info/221bc9e3b8f9473188be9661a5d011fe is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732387663916.e2564e420fa6e11419482705c96414e0./info:regioninfo/1732387668050/Put/seqid=0 2024-11-23T18:48:09,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741943_1123 (size=6778) 2024-11-23T18:48:09,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741943_1123 (size=6778) 2024-11-23T18:48:09,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741943_1123 (size=6778) 2024-11-23T18:48:09,037 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/info/221bc9e3b8f9473188be9661a5d011fe 2024-11-23T18:48:09,042 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/.tmp/info/221bc9e3b8f9473188be9661a5d011fe as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/info/221bc9e3b8f9473188be9661a5d011fe 2024-11-23T18:48:09,047 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/info/221bc9e3b8f9473188be9661a5d011fe, entries=8, sequenceid=23, filesize=6.6 K 2024-11-23T18:48:09,048 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-23T18:48:09,048 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T18:48:09,053 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-23T18:48:09,054 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T18:48:09,054 INFO [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T18:48:09,054 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732387689024Running coprocessor pre-close hooks at 1732387689024Disabling compacts and flushes for region at 1732387689024Disabling writes for close at 1732387689024Obtaining lock to block concurrent updates at 1732387689025 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732387689025Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732387689025Flushing stores of hbase:meta,,1.1588230740 at 1732387689026 (+1 ms)Flushing 1588230740/info: creating writer at 1732387689026Flushing 1588230740/info: appending metadata at 1732387689031 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732387689031Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f1fc8ac: reopening flushed file at 1732387689041 (+10 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1732387689048 (+7 ms)Writing region close event to WAL at 1732387689050 (+2 ms)Running coprocessor post-close hooks at 1732387689054 (+4 ms)Closed at 1732387689054 2024-11-23T18:48:09,054 DEBUG [RS_CLOSE_META-regionserver/2eda60943ea1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T18:48:09,068 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:48:09,068 INFO [regionserver/2eda60943ea1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:48:09,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,37545,1732387649456 2024-11-23T18:48:09,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:48:09,076 INFO [RS:1;2eda60943ea1:37545 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:48:09,086 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,37545,1732387649456] 2024-11-23T18:48:09,097 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,37545,1732387649456 already deleted, retry=false 2024-11-23T18:48:09,097 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,37545,1732387649456 expired; onlineServers=1 2024-11-23T18:48:09,187 INFO [RS:1;2eda60943ea1:37545 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:48:09,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:48:09,187 INFO [RS:1;2eda60943ea1:37545 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,37545,1732387649456; zookeeper connection closed. 2024-11-23T18:48:09,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37545-0x1016905acb30002, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:48:09,187 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f2b5cf2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f2b5cf2 2024-11-23T18:48:09,224 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(976): stopping server 2eda60943ea1,36631,1732387649346; all regions closed. 2024-11-23T18:48:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741893_1071 (size=2422) 2024-11-23T18:48:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741893_1071 (size=2422) 2024-11-23T18:48:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741893_1071 (size=2422) 2024-11-23T18:48:09,233 DEBUG [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs 2024-11-23T18:48:09,233 INFO [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C36631%2C1732387649346.meta:.meta(num 1732387667135) 2024-11-23T18:48:09,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741833_1009 (size=3433) 2024-11-23T18:48:09,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741833_1009 (size=3433) 2024-11-23T18:48:09,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741833_1009 (size=3433) 2024-11-23T18:48:09,238 DEBUG [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/oldWALs 2024-11-23T18:48:09,239 INFO [RS:0;2eda60943ea1:36631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2eda60943ea1%2C36631%2C1732387649346:(num 1732387651084) 2024-11-23T18:48:09,239 DEBUG [RS:0;2eda60943ea1:36631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:48:09,239 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T18:48:09,239 INFO [RS:0;2eda60943ea1:36631 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:48:09,239 INFO [RS:0;2eda60943ea1:36631 {}] hbase.ChoreService(370): Chore service for: regionserver/2eda60943ea1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T18:48:09,239 INFO [RS:0;2eda60943ea1:36631 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:48:09,239 INFO [regionserver/2eda60943ea1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:48:09,239 INFO [RS:0;2eda60943ea1:36631 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36631 2024-11-23T18:48:09,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T18:48:09,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2eda60943ea1,36631,1732387649346 2024-11-23T18:48:09,250 INFO [RS:0;2eda60943ea1:36631 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:48:09,260 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2eda60943ea1,36631,1732387649346] 2024-11-23T18:48:09,271 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2eda60943ea1,36631,1732387649346 already deleted, retry=false 2024-11-23T18:48:09,271 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2eda60943ea1,36631,1732387649346 expired; onlineServers=0 2024-11-23T18:48:09,271 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2eda60943ea1,37311,1732387648561' ***** 2024-11-23T18:48:09,271 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T18:48:09,272 INFO [M:0;2eda60943ea1:37311 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T18:48:09,272 INFO [M:0;2eda60943ea1:37311 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T18:48:09,272 DEBUG [M:0;2eda60943ea1:37311 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T18:48:09,272 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T18:48:09,272 DEBUG [M:0;2eda60943ea1:37311 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T18:48:09,272 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387650656 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.large.0-1732387650656,5,FailOnTimeoutGroup] 2024-11-23T18:48:09,272 DEBUG [master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387650660 {}] cleaner.HFileCleaner(306): Exit Thread[master/2eda60943ea1:0:becomeActiveMaster-HFileCleaner.small.0-1732387650660,5,FailOnTimeoutGroup] 2024-11-23T18:48:09,273 INFO [M:0;2eda60943ea1:37311 {}] hbase.ChoreService(370): Chore service for: master/2eda60943ea1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T18:48:09,273 INFO [M:0;2eda60943ea1:37311 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T18:48:09,273 DEBUG [M:0;2eda60943ea1:37311 {}] master.HMaster(1795): Stopping service threads 2024-11-23T18:48:09,273 INFO [M:0;2eda60943ea1:37311 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T18:48:09,273 INFO [M:0;2eda60943ea1:37311 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T18:48:09,274 INFO [M:0;2eda60943ea1:37311 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T18:48:09,274 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T18:48:09,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:48:09,361 INFO [RS:0;2eda60943ea1:36631 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:48:09,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36631-0x1016905acb30001, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:48:09,361 INFO [RS:0;2eda60943ea1:36631 {}] regionserver.HRegionServer(1031): Exiting; stopping=2eda60943ea1,36631,1732387649346; zookeeper connection closed. 2024-11-23T18:48:09,362 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e81670a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e81670a 2024-11-23T18:48:09,363 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T18:48:09,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T18:48:09,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T18:48:09,366 DEBUG [M:0;2eda60943ea1:37311 {}] zookeeper.ZKUtil(347): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T18:48:09,367 WARN [M:0;2eda60943ea1:37311 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T18:48:09,368 INFO [M:0;2eda60943ea1:37311 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/.lastflushedseqids 2024-11-23T18:48:09,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741944_1124 (size=119) 2024-11-23T18:48:09,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741944_1124 (size=119) 2024-11-23T18:48:09,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741944_1124 (size=119) 2024-11-23T18:48:09,385 INFO [M:0;2eda60943ea1:37311 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T18:48:09,386 INFO [M:0;2eda60943ea1:37311 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T18:48:09,386 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T18:48:09,386 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:48:09,386 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:48:09,386 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T18:48:09,386 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:48:09,386 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-23T18:48:09,403 DEBUG [M:0;2eda60943ea1:37311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7366a7009676407f85f9ecad3505d802 is 82, key is hbase:meta,,1/info:regioninfo/1732387667297/Put/seqid=0 2024-11-23T18:48:09,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741945_1125 (size=6063) 2024-11-23T18:48:09,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741945_1125 (size=6063) 2024-11-23T18:48:09,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741945_1125 (size=6063) 2024-11-23T18:48:09,411 INFO [M:0;2eda60943ea1:37311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7366a7009676407f85f9ecad3505d802 2024-11-23T18:48:09,429 DEBUG [M:0;2eda60943ea1:37311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a2875d4c8d4b40aa9bbdedb40a778d51 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732387664344/Put/seqid=0 2024-11-23T18:48:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741946_1126 (size=7907) 2024-11-23T18:48:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741946_1126 (size=7907) 2024-11-23T18:48:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741946_1126 (size=7907) 2024-11-23T18:48:09,436 INFO [M:0;2eda60943ea1:37311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a2875d4c8d4b40aa9bbdedb40a778d51 2024-11-23T18:48:09,440 INFO [M:0;2eda60943ea1:37311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a2875d4c8d4b40aa9bbdedb40a778d51 2024-11-23T18:48:09,455 DEBUG [M:0;2eda60943ea1:37311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a3516c3e7a14911b6a5225241766261 is 69, key is 2eda60943ea1,36631,1732387649346/rs:state/1732387650800/Put/seqid=0 2024-11-23T18:48:09,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741947_1127 (size=5440) 2024-11-23T18:48:09,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741947_1127 (size=5440) 2024-11-23T18:48:09,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741947_1127 (size=5440) 2024-11-23T18:48:09,462 INFO [M:0;2eda60943ea1:37311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a3516c3e7a14911b6a5225241766261 2024-11-23T18:48:09,466 INFO [M:0;2eda60943ea1:37311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9a3516c3e7a14911b6a5225241766261 2024-11-23T18:48:09,467 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7366a7009676407f85f9ecad3505d802 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7366a7009676407f85f9ecad3505d802 2024-11-23T18:48:09,472 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7366a7009676407f85f9ecad3505d802, entries=14, sequenceid=207, filesize=5.9 K 2024-11-23T18:48:09,473 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a2875d4c8d4b40aa9bbdedb40a778d51 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a2875d4c8d4b40aa9bbdedb40a778d51 2024-11-23T18:48:09,478 INFO [M:0;2eda60943ea1:37311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a2875d4c8d4b40aa9bbdedb40a778d51 2024-11-23T18:48:09,478 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a2875d4c8d4b40aa9bbdedb40a778d51, entries=21, sequenceid=207, filesize=7.7 K 2024-11-23T18:48:09,480 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a3516c3e7a14911b6a5225241766261 as hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9a3516c3e7a14911b6a5225241766261 2024-11-23T18:48:09,485 INFO [M:0;2eda60943ea1:37311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9a3516c3e7a14911b6a5225241766261 2024-11-23T18:48:09,485 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45809/user/jenkins/test-data/0ed9167d-02a2-baef-abb9-de3120b58f8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9a3516c3e7a14911b6a5225241766261, entries=3, sequenceid=207, filesize=5.3 K 2024-11-23T18:48:09,486 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=207, compaction requested=false 2024-11-23T18:48:09,488 INFO [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T18:48:09,488 DEBUG [M:0;2eda60943ea1:37311 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732387689386Disabling compacts and flushes for region at 1732387689386Disabling writes for close at 1732387689386Obtaining lock to block concurrent updates at 1732387689386Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732387689386Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732387689386Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732387689387 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732387689387Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732387689403 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732387689403Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732387689415 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732387689429 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732387689429Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732387689441 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732387689455 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732387689455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dfc0194: reopening flushed file at 1732387689466 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65727fc0: reopening flushed file at 1732387689472 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f8da2af: reopening flushed file at 1732387689479 (+7 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=207, compaction requested=false at 1732387689486 (+7 ms)Writing region close event to WAL at 1732387689488 (+2 ms)Closed at 1732387689488 2024-11-23T18:48:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34245 is added to blk_1073741830_1006 (size=99436) 2024-11-23T18:48:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39151 is added to blk_1073741830_1006 (size=99436) 2024-11-23T18:48:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38589 is added to blk_1073741830_1006 (size=99436) 2024-11-23T18:48:09,491 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T18:48:09,491 INFO [M:0;2eda60943ea1:37311 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T18:48:09,491 INFO [M:0;2eda60943ea1:37311 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37311 2024-11-23T18:48:09,492 INFO [M:0;2eda60943ea1:37311 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T18:48:09,618 INFO [M:0;2eda60943ea1:37311 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T18:48:09,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:48:09,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37311-0x1016905acb30000, quorum=127.0.0.1:63996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T18:48:09,628 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387678187 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387678187 (inode 16688) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732387668939/wal.1732387678187 (inode 16688) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:48:09,631 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668784 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668784 (inode 16665) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732387668528/wal.1732387668784 (inode 16665) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:48:09,632 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732387659516/wal.1732387659624 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:09,634 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387679309 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387679309 (inode 16714) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732387678816/wal.1732387679309 (inode 16714) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:48:09,634 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732387679461/wal.1732387679531 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:09,635 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732387684291/wal.1732387684500 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:09,635 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732387652754/wal.1732387652925 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T18:48:09,637 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal.1732387652623 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal.1732387652623 (inode 16485) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732387652453/wal.1732387652623 (inode 16485) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:48:09,639 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal.1732387652171 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal.1732387652171 (inode 16462) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732387651941/wal.1732387652171 (inode 16462) Holder DFSClient_NONMAPREDUCE_-922606282_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T18:48:09,643 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ed6f3ed{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:48:09,645 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f154eb3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:48:09,646 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:48:09,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@147d1a17{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:48:09,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@626fc3bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,STOPPED} 2024-11-23T18:48:09,648 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:48:09,648 WARN [BP-657037303-172.17.0.2-1732387644132 heartbeating to localhost/127.0.0.1:45809 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:48:09,648 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:48:09,649 WARN [BP-657037303-172.17.0.2-1732387644132 heartbeating to localhost/127.0.0.1:45809 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-657037303-172.17.0.2-1732387644132 (Datanode Uuid 062048f1-dc65-4c1b-8b20-b35c20c68693) service to localhost/127.0.0.1:45809 2024-11-23T18:48:09,650 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data5/current/BP-657037303-172.17.0.2-1732387644132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:48:09,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data6/current/BP-657037303-172.17.0.2-1732387644132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:48:09,651 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:48:09,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cc5338b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:48:09,656 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1364e1fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:48:09,656 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:48:09,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4925ca3e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:48:09,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2599967d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,STOPPED} 2024-11-23T18:48:09,658 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:48:09,658 WARN [BP-657037303-172.17.0.2-1732387644132 heartbeating to localhost/127.0.0.1:45809 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:48:09,658 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:48:09,658 WARN [BP-657037303-172.17.0.2-1732387644132 heartbeating to localhost/127.0.0.1:45809 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-657037303-172.17.0.2-1732387644132 (Datanode Uuid c4ccde46-6de8-4e29-807c-8c9d02ca3b86) service to localhost/127.0.0.1:45809 2024-11-23T18:48:09,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data3/current/BP-657037303-172.17.0.2-1732387644132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:48:09,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data4/current/BP-657037303-172.17.0.2-1732387644132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:48:09,659 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:48:09,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@516a39dc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T18:48:09,665 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@428881d6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:48:09,665 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:48:09,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c68adb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:48:09,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@302ee775{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,STOPPED} 2024-11-23T18:48:09,667 WARN [BP-657037303-172.17.0.2-1732387644132 heartbeating to localhost/127.0.0.1:45809 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T18:48:09,667 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T18:48:09,667 WARN [BP-657037303-172.17.0.2-1732387644132 heartbeating to localhost/127.0.0.1:45809 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-657037303-172.17.0.2-1732387644132 (Datanode Uuid 82f019bb-94fc-4511-8a11-4ed7272f1081) service to localhost/127.0.0.1:45809 2024-11-23T18:48:09,667 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T18:48:09,667 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data1/current/BP-657037303-172.17.0.2-1732387644132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:48:09,668 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/cluster_3c285ca5-2169-6344-96b0-7528fb55bc4e/data/data2/current/BP-657037303-172.17.0.2-1732387644132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T18:48:09,668 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T18:48:09,675 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fe9bfd3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T18:48:09,676 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16695a38{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T18:48:09,676 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T18:48:09,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d655053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T18:48:09,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b5fa2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7608deac-1be9-8de6-bc58-0542358c28d8/hadoop.log.dir/,STOPPED} 2024-11-23T18:48:09,686 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T18:48:09,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down